// SPDX-License-Identifier: GPL-2.0/* * linux/ipc/shm.c * Copyright (C) 1992, 1993 Krishna Balasubramanian *	 Many improvements/fixes by Bruno Haible. * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994. * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli. * * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com> * BIGMEM support, Andrea Arcangeli <andrea@suse.de> * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr> * HIGHMEM support, Ingo Molnar <mingo@redhat.com> * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com> * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com> * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com> * * support for audit of ipc object properties and permission changes * Dustin Kirkland <dustin.kirkland@us.ibm.com> * * namespaces support * OpenVZ, SWsoft Inc. * Pavel Emelianov <xemul@openvz.org> * * Better ipc lock (kern_ipc_perm.lock) handling * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013. */ #include <linux/slab.h>#include <linux/mm.h>#include <linux/hugetlb.h>#include <linux/shm.h>#include <uapi/linux/shm.h>#include <linux/init.h>#include <linux/file.h>#include <linux/mman.h>#include <linux/shmem_fs.h>#include <linux/security.h>#include <linux/syscalls.h>#include <linux/audit.h>#include <linux/capability.h>#include <linux/ptrace.h>#include <linux/seq_file.h>#include <linux/rwsem.h>#include <linux/nsproxy.h>#include <linux/mount.h>#include <linux/ipc_namespace.h>#include <linux/rhashtable.h>#include <linux/nstree.h> #include <linux/uaccess.h> #include "util.h" struct shmid_kernel /* private to the kernel */{	struct kern_ipc_perm	shm_perm;	struct file		*shm_file;	unsigned long		shm_nattch;	unsigned long		shm_segsz;	time64_t		shm_atim;	time64_t		shm_dtim;	time64_t		shm_ctim;	struct pid		*shm_cprid;	struct pid		*shm_lprid;	struct ucounts		*mlock_ucounts; 	/*	 * The task created the shm object, for	 * task_lock(shp->shm_creator)	 */	struct task_struct	*shm_creator; 	/*	 * List by creator. task_lock(->shm_creator) required for read/write.	 * If list_empty(), then the creator is dead already.	 */	struct list_head	shm_clist;	struct ipc_namespace	*ns;} __randomize_layout; /* shm_mode upper byte flags */#define SHM_DEST	01000	/* segment will be destroyed on last detach */#define SHM_LOCKED	02000   /* segment will not be swapped */ struct shm_file_data {	int id;	struct ipc_namespace *ns;	struct file *file;	const struct vm_operations_struct *vm_ops;}; #define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data)) static const struct file_operations shm_file_operations;static const struct vm_operations_struct shm_vm_ops; #define shm_ids(ns)	((ns)->ids[IPC_SHM_IDS]) #define shm_unlock(shp)			\	ipc_unlock(&(shp)->shm_perm) static int newseg(struct ipc_namespace *, struct ipc_params *);static void shm_open(struct vm_area_struct *vma);static void shm_close(struct vm_area_struct *vma);static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);#ifdef CONFIG_PROC_FSstatic int sysvipc_shm_proc_show(struct seq_file *s, void *it);#endif void shm_init_ns(struct ipc_namespace *ns){	ns->shm_ctlmax = SHMMAX;	ns->shm_ctlall = SHMALL;	ns->shm_ctlmni = SHMMNI;	ns->shm_rmid_forced = 0;	ns->shm_tot = 0;	ipc_init_ids(&shm_ids(ns));} /* * Called with shm_ids.rwsem (writer) and the shp structure locked. * Only shm_ids.rwsem remains locked on exit. */static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp){	struct shmid_kernel *shp; 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);	WARN_ON(ns != shp->ns); 	if (shp->shm_nattch) {		shp->shm_perm.mode |= SHM_DEST;		/* Do not find it any more */		ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);		shm_unlock(shp);	} else		shm_destroy(ns, shp);} #ifdef CONFIG_IPC_NSvoid shm_exit_ns(struct ipc_namespace *ns){	free_ipcs(ns, &shm_ids(ns), do_shm_rmid);	idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);	rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);}#endif static int __init ipc_ns_init(void){	shm_init_ns(&init_ipc_ns);	ns_tree_add(&init_ipc_ns);	return 0;} pure_initcall(ipc_ns_init); void __init shm_init(void){	ipc_init_proc_interface("sysvipc/shm",#if BITS_PER_LONG <= 32				"       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",#else				"       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",#endif				IPC_SHM_IDS, sysvipc_shm_proc_show);} static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id){	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id); 	if (IS_ERR(ipcp))		return ERR_CAST(ipcp); 	return container_of(ipcp, struct shmid_kernel, shm_perm);} static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id){	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id); 	if (IS_ERR(ipcp))		return ERR_CAST(ipcp); 	return container_of(ipcp, struct shmid_kernel, shm_perm);} /* * shm_lock_(check_) routines are called in the paths where the rwsem * is not necessarily held. */static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id){	struct kern_ipc_perm *ipcp; 	rcu_read_lock();	ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);	if (IS_ERR(ipcp))		goto err; 	ipc_lock_object(ipcp);	/*	 * ipc_rmid() may have already freed the ID while ipc_lock_object()	 * was spinning: here verify that the structure is still valid.	 * Upon races with RMID, return -EIDRM, thus indicating that	 * the ID points to a removed identifier.	 */	if (ipc_valid_object(ipcp)) {		/* return a locked ipc object upon success */		return container_of(ipcp, struct shmid_kernel, shm_perm);	} 	ipc_unlock_object(ipcp);	ipcp = ERR_PTR(-EIDRM);err:	rcu_read_unlock();	/*	 * Callers of shm_lock() must validate the status of the returned ipc	 * object pointer and error out as appropriate.	 */	return ERR_CAST(ipcp);} static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp){	rcu_read_lock();	ipc_lock_object(&ipcp->shm_perm);} static void shm_rcu_free(struct rcu_head *head){	struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,							rcu);	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,							shm_perm);	security_shm_free(&shp->shm_perm);	kfree(shp);} /* * It has to be called with shp locked. * It must be called before ipc_rmid() */static inline void shm_clist_rm(struct shmid_kernel *shp){	struct task_struct *creator; 	/* ensure that shm_creator does not disappear */	rcu_read_lock(); 	/*	 * A concurrent exit_shm may do a list_del_init() as well.	 * Just do nothing if exit_shm already did the work	 */	if (!list_empty(&shp->shm_clist)) {		/*		 * shp->shm_creator is guaranteed to be valid *only*		 * if shp->shm_clist is not empty.		 */		creator = shp->shm_creator; 		task_lock(creator);		/*		 * list_del_init() is a nop if the entry was already removed		 * from the list.		 */		list_del_init(&shp->shm_clist);		task_unlock(creator);	}	rcu_read_unlock();} static inline void shm_rmid(struct shmid_kernel *s){	shm_clist_rm(s);	ipc_rmid(&shm_ids(s->ns), &s->shm_perm);}  static int __shm_open(struct shm_file_data *sfd){	struct shmid_kernel *shp; 	shp = shm_lock(sfd->ns, sfd->id); 	if (IS_ERR(shp))		return PTR_ERR(shp); 	if (shp->shm_file != sfd->file) {		/* ID was reused */		shm_unlock(shp);		return -EINVAL;	} 	shp->shm_atim = ktime_get_real_seconds();	ipc_update_pid(&shp->shm_lprid, task_tgid(current));	shp->shm_nattch++;	shm_unlock(shp);	return 0;} /* This is called by fork, once for every shm attach. */static void shm_open(struct vm_area_struct *vma){	struct file *file = vma->vm_file;	struct shm_file_data *sfd = shm_file_data(file);	int err; 	/* Always call underlying open if present */	if (sfd->vm_ops->open)		sfd->vm_ops->open(vma); 	err = __shm_open(sfd);	/*	 * We raced in the idr lookup or with shm_destroy().	 * Either way, the ID is busted.	 */	WARN_ON_ONCE(err);} /* * shm_destroy - free the struct shmid_kernel * * @ns: namespace * @shp: struct to free * * It has to be called with shp and shm_ids.rwsem (writer) locked, * but returns with shp unlocked and freed. */static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp){	struct file *shm_file; 	shm_file = shp->shm_file;	shp->shm_file = NULL;	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;	shm_rmid(shp);	shm_unlock(shp);	if (!is_file_hugepages(shm_file))		shmem_lock(shm_file, 0, shp->mlock_ucounts);	fput(shm_file);	ipc_update_pid(&shp->shm_cprid, NULL);	ipc_update_pid(&shp->shm_lprid, NULL);	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);} /* * shm_may_destroy - identifies whether shm segment should be destroyed now * * Returns true if and only if there are no active users of the segment and * one of the following is true: * * 1) shmctl(id, IPC_RMID, NULL) was called for this shp * * 2) sysctl kernel.shm_rmid_forced is set to 1. */static bool shm_may_destroy(struct shmid_kernel *shp){	return (shp->shm_nattch == 0) &&	       (shp->ns->shm_rmid_forced ||		(shp->shm_perm.mode & SHM_DEST));} /* * remove the attach descriptor vma. * free memory for segment if it is marked destroyed. * The descriptor has already been removed from the current->mm->mmap list * and will later be kfree()d. */static void __shm_close(struct shm_file_data *sfd){	struct shmid_kernel *shp;	struct ipc_namespace *ns = sfd->ns; 	down_write(&shm_ids(ns).rwsem);	/* remove from the list of attaches of the shm segment */	shp = shm_lock(ns, sfd->id); 	/*	 * We raced in the idr lookup or with shm_destroy().	 * Either way, the ID is busted.	 */	if (WARN_ON_ONCE(IS_ERR(shp)))		goto done; /* no-op */ 	ipc_update_pid(&shp->shm_lprid, task_tgid(current));	shp->shm_dtim = ktime_get_real_seconds();	shp->shm_nattch--;	if (shm_may_destroy(shp))		shm_destroy(ns, shp);	else		shm_unlock(shp);done:	up_write(&shm_ids(ns).rwsem);} static void shm_close(struct vm_area_struct *vma){	struct file *file = vma->vm_file;	struct shm_file_data *sfd = shm_file_data(file); 	/* Always call underlying close if present */	if (sfd->vm_ops->close)		sfd->vm_ops->close(vma); 	__shm_close(sfd);} /* Called with ns->shm_ids(ns).rwsem locked */static int shm_try_destroy_orphaned(int id, void *p, void *data){	struct ipc_namespace *ns = data;	struct kern_ipc_perm *ipcp = p;	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm); 	/*	 * We want to destroy segments without users and with already	 * exit'ed originating process.	 *	 * shm_nattch can be changed under shm_perm.lock without holding the	 * rwsem, so take the object lock before checking shm_may_destroy().	 */	if (!list_empty(&shp->shm_clist))		return 0; 	shm_lock_by_ptr(shp);	if (shm_may_destroy(shp))		shm_destroy(ns, shp);	else		shm_unlock(shp);	return 0;} void shm_destroy_orphaned(struct ipc_namespace *ns){	down_write(&shm_ids(ns).rwsem);	if (shm_ids(ns).in_use) {		rcu_read_lock();		idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);		rcu_read_unlock();	}	up_write(&shm_ids(ns).rwsem);} /* Locking assumes this will only be called with task == current */void exit_shm(struct task_struct *task){	for (;;) {		struct shmid_kernel *shp;		struct ipc_namespace *ns; 		task_lock(task); 		if (list_empty(&task->sysvshm.shm_clist)) {			task_unlock(task);			break;		} 		shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,				shm_clist); 		/*		 * 1) Get pointer to the ipc namespace. It is worth to say		 * that this pointer is guaranteed to be valid because		 * shp lifetime is always shorter than namespace lifetime		 * in which shp lives.		 * We taken task_lock it means that shp won't be freed.		 */		ns = shp->ns; 		/*		 * 2) If kernel.shm_rmid_forced is not set then only keep track of		 * which shmids are orphaned, so that a later set of the sysctl		 * can clean them up.		 */		if (!ns->shm_rmid_forced)			goto unlink_continue; 		/*		 * 3) get a reference to the namespace.		 *    The refcount could be already 0. If it is 0, then		 *    the shm objects will be free by free_ipc_work().		 */		ns = get_ipc_ns_not_zero(ns);		if (!ns) {unlink_continue:			list_del_init(&shp->shm_clist);			task_unlock(task);			continue;		} 		/*		 * 4) get a reference to shp.		 *   This cannot fail: shm_clist_rm() is called before		 *   ipc_rmid(), thus the refcount cannot be 0.		 */		WARN_ON(!ipc_rcu_getref(&shp->shm_perm)); 		/*

Lines 1–500 of 1883. Load more