/* $NetBSD: sysv_shm.c,v 1.135.2.5 2024/03/11 18:01:48 martin Exp $ */ /*- * Copyright (c) 1999, 2007 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center, and by Mindaugas Rasiukevicius. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 1994 Adam Glass and Charles M. Hannum. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Adam Glass and Charles M. * Hannum. * 4. The names of the authors may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __KERNEL_RCSID(0, "$NetBSD: sysv_shm.c,v 1.135.2.5 2024/03/11 18:01:48 martin Exp $"); #ifdef _KERNEL_OPT #include "opt_sysv.h" #endif #include #include #include #include #include #include #include #include #include /* XXX for */ #include #include #include #include #include struct shmmap_entry { SLIST_ENTRY(shmmap_entry) next; vaddr_t va; int shmid; }; int shm_nused __cacheline_aligned; struct shmid_ds * shmsegs __read_mostly; static kmutex_t shm_lock __cacheline_aligned; static kcondvar_t * shm_cv __cacheline_aligned; static int shm_last_free __cacheline_aligned; static size_t shm_committed __cacheline_aligned; static int shm_use_phys __read_mostly; static kcondvar_t shm_realloc_cv; static bool shm_realloc_state; static u_int shm_realloc_disable; struct shmmap_state { unsigned int nitems; unsigned int nrefs; SLIST_HEAD(, shmmap_entry) entries; }; extern int kern_has_sysvshm; SYSCTL_SETUP_PROTO(sysctl_ipc_shm_setup); #ifdef SHMDEBUG #define SHMPRINTF(a) printf a #else #define SHMPRINTF(a) #endif static int shmrealloc(int); /* * Find the shared memory segment permission by the index. Only used by * compat_linux to implement SHM_STAT. */ int shm_find_segment_perm_by_index(int index, struct ipc_perm *perm) { struct shmid_ds *shmseg; mutex_enter(&shm_lock); if (index < 0 || index >= shminfo.shmmni) { mutex_exit(&shm_lock); return EINVAL; } shmseg = &shmsegs[index]; memcpy(perm, &shmseg->shm_perm, sizeof(*perm)); mutex_exit(&shm_lock); return 0; } /* * Find the shared memory segment by the identifier. * => must be called with shm_lock held; */ static struct shmid_ds * shm_find_segment_by_shmid(int shmid) { int segnum; struct shmid_ds *shmseg; KASSERT(mutex_owned(&shm_lock)); segnum = IPCID_TO_IX(shmid); if (segnum < 0 || segnum >= shminfo.shmmni) return NULL; shmseg = &shmsegs[segnum]; if ((shmseg->shm_perm.mode & SHMSEG_ALLOCATED) == 0) return NULL; if ((shmseg->shm_perm.mode & (SHMSEG_REMOVED|SHMSEG_RMLINGER)) == SHMSEG_REMOVED) return NULL; if (shmseg->shm_perm._seq != IPCID_TO_SEQ(shmid)) return NULL; return shmseg; } /* * Free memory segment. * => must be called with shm_lock held; */ static void shm_free_segment(int segnum) { struct shmid_ds *shmseg; size_t size; bool wanted; KASSERT(mutex_owned(&shm_lock)); shmseg = &shmsegs[segnum]; SHMPRINTF(("shm freeing key 0x%lx seq 0x%x\n", shmseg->shm_perm._key, shmseg->shm_perm._seq)); size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; wanted = (shmseg->shm_perm.mode & SHMSEG_WANTED); shmseg->_shm_internal = NULL; shm_committed -= btoc(size); shm_nused--; shmseg->shm_perm.mode = SHMSEG_FREE; shm_last_free = segnum; if (wanted == true) cv_broadcast(&shm_cv[segnum]); } /* * Delete entry from the shm map. * => must be called with shm_lock held; */ static struct uvm_object * shm_delete_mapping(struct shmmap_state *shmmap_s, struct shmmap_entry *shmmap_se) { struct uvm_object *uobj = NULL; struct shmid_ds *shmseg; int segnum; KASSERT(mutex_owned(&shm_lock)); segnum = IPCID_TO_IX(shmmap_se->shmid); shmseg = &shmsegs[segnum]; SLIST_REMOVE(&shmmap_s->entries, shmmap_se, shmmap_entry, next); shmmap_s->nitems--; shmseg->shm_dtime = time_second; if ((--shmseg->shm_nattch <= 0) && (shmseg->shm_perm.mode & SHMSEG_REMOVED)) { uobj = shmseg->_shm_internal; shm_free_segment(segnum); } return uobj; } /* * Get a non-shared shm map for that vmspace. Note, that memory * allocation might be performed with lock held. */ static struct shmmap_state * shmmap_getprivate(struct proc *p) { struct shmmap_state *oshmmap_s, *shmmap_s; struct shmmap_entry *oshmmap_se, *shmmap_se; KASSERT(mutex_owned(&shm_lock)); /* 1. A shm map with refcnt = 1, used by ourselves, thus return */ oshmmap_s = (struct shmmap_state *)p->p_vmspace->vm_shm; if (oshmmap_s && oshmmap_s->nrefs == 1) return oshmmap_s; /* 2. No shm map preset - create a fresh one */ shmmap_s = kmem_zalloc(sizeof(struct shmmap_state), KM_SLEEP); shmmap_s->nrefs = 1; SLIST_INIT(&shmmap_s->entries); p->p_vmspace->vm_shm = (void *)shmmap_s; if (oshmmap_s == NULL) return shmmap_s; SHMPRINTF(("shmmap_getprivate: vm %p split (%d entries), was used by %d\n", p->p_vmspace, oshmmap_s->nitems, oshmmap_s->nrefs)); /* 3. A shared shm map, copy to a fresh one and adjust refcounts */ SLIST_FOREACH(oshmmap_se, &oshmmap_s->entries, next) { shmmap_se = kmem_alloc(sizeof(struct shmmap_entry), KM_SLEEP); shmmap_se->va = oshmmap_se->va; shmmap_se->shmid = oshmmap_se->shmid; SLIST_INSERT_HEAD(&shmmap_s->entries, shmmap_se, next); } shmmap_s->nitems = oshmmap_s->nitems; oshmmap_s->nrefs--; return shmmap_s; } /* * Lock/unlock the memory. * => must be called with shm_lock held; */ static int shm_memlock(struct shmid_ds *shmseg, int shmid, int cmd) { size_t size; int error; KASSERT(mutex_owned(&shm_lock)); size = round_page(shmseg->shm_segsz); if (cmd == SHM_LOCK && (shmseg->shm_perm.mode & SHMSEG_WIRED) == 0) { /* Wire the object and map, then tag it */ error = uvm_obj_wirepages(shmseg->_shm_internal, 0, size, NULL); if (error) return EIO; shmseg->shm_perm.mode |= SHMSEG_WIRED; } else if (cmd == SHM_UNLOCK && (shmseg->shm_perm.mode & SHMSEG_WIRED) != 0) { /* Unwire the object, then untag it */ uvm_obj_unwirepages(shmseg->_shm_internal, 0, size); shmseg->shm_perm.mode &= ~SHMSEG_WIRED; } return 0; } /* * Unmap shared memory. */ int sys_shmdt(struct lwp *l, const struct sys_shmdt_args *uap, register_t *retval) { /* { syscallarg(const void *) shmaddr; } */ struct proc *p = l->l_proc; struct shmmap_state *shmmap_s1, *shmmap_s; struct shmmap_entry *shmmap_se; struct uvm_object *uobj; struct shmid_ds *shmseg; size_t size; mutex_enter(&shm_lock); /* In case of reallocation, we will wait for completion */ while (__predict_false(shm_realloc_state)) cv_wait(&shm_realloc_cv, &shm_lock); shmmap_s1 = (struct shmmap_state *)p->p_vmspace->vm_shm; if (shmmap_s1 == NULL) { mutex_exit(&shm_lock); return EINVAL; } /* Find the map entry */ SLIST_FOREACH(shmmap_se, &shmmap_s1->entries, next) if (shmmap_se->va == (vaddr_t)SCARG(uap, shmaddr)) break; if (shmmap_se == NULL) { mutex_exit(&shm_lock); return EINVAL; } shmmap_s = shmmap_getprivate(p); if (shmmap_s != shmmap_s1) { /* Map has been copied, lookup entry in new map */ SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) if (shmmap_se->va == (vaddr_t)SCARG(uap, shmaddr)) break; if (shmmap_se == NULL) { mutex_exit(&shm_lock); return EINVAL; } } SHMPRINTF(("shmdt: vm %p: remove %d @%lx\n", p->p_vmspace, shmmap_se->shmid, shmmap_se->va)); /* Delete the entry from shm map */ uobj = shm_delete_mapping(shmmap_s, shmmap_se); shmseg = &shmsegs[IPCID_TO_IX(shmmap_se->shmid)]; size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; mutex_exit(&shm_lock); uvm_deallocate(&p->p_vmspace->vm_map, shmmap_se->va, size); if (uobj != NULL) { uao_detach(uobj); } kmem_free(shmmap_se, sizeof(struct shmmap_entry)); return 0; } /* * Map shared memory. */ int sys_shmat(struct lwp *l, const struct sys_shmat_args *uap, register_t *retval) { /* { syscallarg(int) shmid; syscallarg(const void *) shmaddr; syscallarg(int) shmflg; } */ int error, flags = 0; struct proc *p = l->l_proc; kauth_cred_t cred = l->l_cred; struct shmid_ds *shmseg; struct shmmap_state *shmmap_s; struct shmmap_entry *shmmap_se; struct uvm_object *uobj; struct vmspace *vm; vaddr_t attach_va; vm_prot_t prot; vsize_t size; /* Allocate a new map entry and set it */ shmmap_se = kmem_alloc(sizeof(struct shmmap_entry), KM_SLEEP); shmmap_se->shmid = SCARG(uap, shmid); mutex_enter(&shm_lock); /* In case of reallocation, we will wait for completion */ while (__predict_false(shm_realloc_state)) cv_wait(&shm_realloc_cv, &shm_lock); shmseg = shm_find_segment_by_shmid(SCARG(uap, shmid)); if (shmseg == NULL) { error = EINVAL; goto err; } error = ipcperm(cred, &shmseg->shm_perm, (SCARG(uap, shmflg) & SHM_RDONLY) ? IPC_R : IPC_R|IPC_W); if (error) goto err; vm = p->p_vmspace; shmmap_s = (struct shmmap_state *)vm->vm_shm; if (shmmap_s && shmmap_s->nitems >= shminfo.shmseg) { error = EMFILE; goto err; } size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; prot = VM_PROT_READ; if ((SCARG(uap, shmflg) & SHM_RDONLY) == 0) prot |= VM_PROT_WRITE; if (SCARG(uap, shmaddr)) { flags |= UVM_FLAG_FIXED; if (SCARG(uap, shmflg) & SHM_RND) attach_va = (vaddr_t)SCARG(uap, shmaddr) & ~(SHMLBA-1); else if (((vaddr_t)SCARG(uap, shmaddr) & (SHMLBA-1)) == 0) attach_va = (vaddr_t)SCARG(uap, shmaddr); else { error = EINVAL; goto err; } } else { /* This is just a hint to uvm_map() about where to put it. */ attach_va = p->p_emul->e_vm_default_addr(p, (vaddr_t)vm->vm_daddr, size, p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN); } /* * Create a map entry, add it to the list and increase the counters. */ shmmap_s = shmmap_getprivate(p); SLIST_INSERT_HEAD(&shmmap_s->entries, shmmap_se, next); shmmap_s->nitems++; shmseg->shm_lpid = p->p_pid; shmseg->shm_nattch++; /* * Map the segment into the address space. */ uobj = shmseg->_shm_internal; uao_reference(uobj); error = uvm_map(&vm->vm_map, &attach_va, size, uobj, 0, 0, UVM_MAPFLAG(prot, prot, UVM_INH_SHARE, UVM_ADV_RANDOM, flags)); if (error) goto err_detach; /* Set the new address, and update the time */ shmmap_se->va = attach_va; shmseg->shm_atime = time_second; retval[0] = attach_va; SHMPRINTF(("shmat: vm %p: add %d @%lx\n", p->p_vmspace, shmmap_se->shmid, attach_va)); err: mutex_exit(&shm_lock); if (error && shmmap_se) { kmem_free(shmmap_se, sizeof(struct shmmap_entry)); } return error; err_detach: uao_detach(uobj); uobj = shm_delete_mapping(shmmap_s, shmmap_se); mutex_exit(&shm_lock); if (uobj != NULL) { uao_detach(uobj); } kmem_free(shmmap_se, sizeof(struct shmmap_entry)); return error; } /* * Shared memory control operations. */ int sys___shmctl50(struct lwp *l, const struct sys___shmctl50_args *uap, register_t *retval) { /* { syscallarg(int) shmid; syscallarg(int) cmd; syscallarg(struct shmid_ds *) buf; } */ struct shmid_ds shmbuf; int cmd, error; cmd = SCARG(uap, cmd); if (cmd == IPC_SET) { error = copyin(SCARG(uap, buf), &shmbuf, sizeof(shmbuf)); if (error) return error; } error = shmctl1(l, SCARG(uap, shmid), cmd, (cmd == IPC_SET || cmd == IPC_STAT) ? &shmbuf : NULL); if (error == 0 && cmd == IPC_STAT) error = copyout(&shmbuf, SCARG(uap, buf), sizeof(shmbuf)); return error; } int shmctl1(struct lwp *l, int shmid, int cmd, struct shmid_ds *shmbuf) { struct uvm_object *uobj = NULL; kauth_cred_t cred = l->l_cred; struct shmid_ds *shmseg; int error = 0; mutex_enter(&shm_lock); /* In case of reallocation, we will wait for completion */ while (__predict_false(shm_realloc_state)) cv_wait(&shm_realloc_cv, &shm_lock); shmseg = shm_find_segment_by_shmid(shmid); if (shmseg == NULL) { mutex_exit(&shm_lock); return EINVAL; } switch (cmd) { case IPC_STAT: if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_R)) != 0) break; memset(shmbuf, 0, sizeof *shmbuf); shmbuf->shm_perm = shmseg->shm_perm; shmbuf->shm_perm.mode &= 0777; shmbuf->shm_segsz = shmseg->shm_segsz; shmbuf->shm_lpid = shmseg->shm_lpid; shmbuf->shm_cpid = shmseg->shm_cpid; shmbuf->shm_nattch = shmseg->shm_nattch; shmbuf->shm_atime = shmseg->shm_atime; shmbuf->shm_dtime = shmseg->shm_dtime; shmbuf->shm_ctime = shmseg->shm_ctime; break; case IPC_SET: if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_M)) != 0) break; shmseg->shm_perm.uid = shmbuf->shm_perm.uid; shmseg->shm_perm.gid = shmbuf->shm_perm.gid; shmseg->shm_perm.mode = (shmseg->shm_perm.mode & ~ACCESSPERMS) | (shmbuf->shm_perm.mode & ACCESSPERMS); shmseg->shm_ctime = time_second; break; case IPC_RMID: if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_M)) != 0) break; shmseg->shm_perm._key = IPC_PRIVATE; shmseg->shm_perm.mode |= SHMSEG_REMOVED; if (shmseg->shm_nattch <= 0) { uobj = shmseg->_shm_internal; shm_free_segment(IPCID_TO_IX(shmid)); } break; case SHM_LOCK: case SHM_UNLOCK: if ((error = kauth_authorize_system(cred, KAUTH_SYSTEM_SYSVIPC, (cmd == SHM_LOCK) ? KAUTH_REQ_SYSTEM_SYSVIPC_SHM_LOCK : KAUTH_REQ_SYSTEM_SYSVIPC_SHM_UNLOCK, NULL, NULL, NULL)) != 0) break; error = shm_memlock(shmseg, shmid, cmd); break; default: error = EINVAL; } mutex_exit(&shm_lock); if (uobj != NULL) uao_detach(uobj); return error; } /* * Try to take an already existing segment. * => must be called with shm_lock held; * => called from one place, thus, inline; */ static inline int shmget_existing(struct lwp *l, const struct sys_shmget_args *uap, int mode, register_t *retval) { struct shmid_ds *shmseg; kauth_cred_t cred = l->l_cred; int segnum, error; again: KASSERT(mutex_owned(&shm_lock)); /* Find segment by key */ for (segnum = 0; segnum < shminfo.shmmni; segnum++) if ((shmsegs[segnum].shm_perm.mode & SHMSEG_ALLOCATED) && shmsegs[segnum].shm_perm._key == SCARG(uap, key)) break; if (segnum == shminfo.shmmni) { /* Not found */ return -1; } shmseg = &shmsegs[segnum]; if (shmseg->shm_perm.mode & SHMSEG_REMOVED) { /* * This segment is in the process of being allocated. Wait * until it's done, and look the key up again (in case the * allocation failed or it was freed). */ shmseg->shm_perm.mode |= SHMSEG_WANTED; error = cv_wait_sig(&shm_cv[segnum], &shm_lock); if (error) return error; goto again; } /* * First check the flags, to generate a useful error when a * segment already exists. */ if ((SCARG(uap, shmflg) & (IPC_CREAT | IPC_EXCL)) == (IPC_CREAT | IPC_EXCL)) return EEXIST; /* Check the permission and segment size. */ error = ipcperm(cred, &shmseg->shm_perm, mode); if (error) return error; if (SCARG(uap, size) && SCARG(uap, size) > shmseg->shm_segsz) return EINVAL; *retval = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm); return 0; } int sys_shmget(struct lwp *l, const struct sys_shmget_args *uap, register_t *retval) { /* { syscallarg(key_t) key; syscallarg(size_t) size; syscallarg(int) shmflg; } */ struct shmid_ds *shmseg; kauth_cred_t cred = l->l_cred; key_t key = SCARG(uap, key); size_t size; int error, mode, segnum; bool lockmem; mode = SCARG(uap, shmflg) & ACCESSPERMS; if (SCARG(uap, shmflg) & _SHM_RMLINGER) mode |= SHMSEG_RMLINGER; SHMPRINTF(("shmget: key 0x%lx size 0x%zx shmflg 0x%x mode 0x%x\n", SCARG(uap, key), SCARG(uap, size), SCARG(uap, shmflg), mode)); mutex_enter(&shm_lock); /* In case of reallocation, we will wait for completion */ while (__predict_false(shm_realloc_state)) cv_wait(&shm_realloc_cv, &shm_lock); if (key != IPC_PRIVATE) { error = shmget_existing(l, uap, mode, retval); if (error != -1) { mutex_exit(&shm_lock); return error; } if ((SCARG(uap, shmflg) & IPC_CREAT) == 0) { mutex_exit(&shm_lock); return ENOENT; } } error = 0; /* * Check the for the limits. */ size = SCARG(uap, size); if (size < shminfo.shmmin || size > shminfo.shmmax) { mutex_exit(&shm_lock); return EINVAL; } if (shm_nused >= shminfo.shmmni) { mutex_exit(&shm_lock); return ENOSPC; } size = round_page(size); if (shm_committed + btoc(size) > shminfo.shmall) { mutex_exit(&shm_lock); return ENOMEM; } /* Find the first available segment */ if (shm_last_free < 0) { for (segnum = 0; segnum < shminfo.shmmni; segnum++) if (shmsegs[segnum].shm_perm.mode & SHMSEG_FREE) break; KASSERT(segnum < shminfo.shmmni); } else { segnum = shm_last_free; shm_last_free = -1; } /* * Initialize the segment. * We will drop the lock while allocating the memory, thus mark the * segment present, but removed, that no other thread could take it. * Also, disable reallocation, while lock is dropped. */ shmseg = &shmsegs[segnum]; shmseg->shm_perm.mode = SHMSEG_ALLOCATED | SHMSEG_REMOVED; shm_committed += btoc(size); shm_nused++; lockmem = shm_use_phys; shm_realloc_disable++; mutex_exit(&shm_lock); /* Allocate the memory object and lock it if needed */ shmseg->_shm_internal = uao_create(size, 0); if (lockmem) { /* Wire the pages and tag it */ error = uvm_obj_wirepages(shmseg->_shm_internal, 0, size, NULL); if (error) { uao_detach(shmseg->_shm_internal); mutex_enter(&shm_lock); shm_free_segment(segnum); shm_realloc_disable--; mutex_exit(&shm_lock); return error; } } /* * Please note, while segment is marked, there are no need to hold the * lock, while setting it (except shm_perm.mode). */ shmseg->shm_perm._key = SCARG(uap, key); shmseg->shm_perm._seq = (shmseg->shm_perm._seq + 1) & 0x7fff; *retval = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm); shmseg->shm_perm.cuid = shmseg->shm_perm.uid = kauth_cred_geteuid(cred); shmseg->shm_perm.cgid = shmseg->shm_perm.gid = kauth_cred_getegid(cred); shmseg->shm_segsz = SCARG(uap, size); shmseg->shm_cpid = l->l_proc->p_pid; shmseg->shm_lpid = shmseg->shm_nattch = 0; shmseg->shm_atime = shmseg->shm_dtime = 0; shmseg->shm_ctime = time_second; /* * Segment is initialized. * Enter the lock, mark as allocated, and notify waiters (if any). * Also, unmark the state of reallocation. */ mutex_enter(&shm_lock); shmseg->shm_perm.mode = (shmseg->shm_perm.mode & SHMSEG_WANTED) | (mode & (ACCESSPERMS | SHMSEG_RMLINGER)) | SHMSEG_ALLOCATED | (lockmem ? SHMSEG_WIRED : 0); if (shmseg->shm_perm.mode & SHMSEG_WANTED) { shmseg->shm_perm.mode &= ~SHMSEG_WANTED; cv_broadcast(&shm_cv[segnum]); } shm_realloc_disable--; cv_broadcast(&shm_realloc_cv); mutex_exit(&shm_lock); return error; } void shmfork(struct vmspace *vm1, struct vmspace *vm2) { struct shmmap_state *shmmap_s; struct shmmap_entry *shmmap_se; SHMPRINTF(("shmfork %p->%p\n", vm1, vm2)); mutex_enter(&shm_lock); vm2->vm_shm = vm1->vm_shm; if (vm1->vm_shm) { shmmap_s = (struct shmmap_state *)vm1->vm_shm; SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) shmsegs[IPCID_TO_IX(shmmap_se->shmid)].shm_nattch++; shmmap_s->nrefs++; } mutex_exit(&shm_lock); } void shmexit(struct vmspace *vm) { struct shmmap_state *shmmap_s; struct shmmap_entry *shmmap_se; mutex_enter(&shm_lock); shmmap_s = (struct shmmap_state *)vm->vm_shm; if (shmmap_s == NULL) { mutex_exit(&shm_lock); return; } vm->vm_shm = NULL; if (--shmmap_s->nrefs > 0) { SHMPRINTF(("shmexit: vm %p drop ref (%d entries), refs = %d\n", vm, shmmap_s->nitems, shmmap_s->nrefs)); SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) { shmsegs[IPCID_TO_IX(shmmap_se->shmid)].shm_nattch--; } mutex_exit(&shm_lock); return; } SHMPRINTF(("shmexit: vm %p cleanup (%d entries)\n", vm, shmmap_s->nitems)); if (shmmap_s->nitems == 0) { mutex_exit(&shm_lock); kmem_free(shmmap_s, sizeof(struct shmmap_state)); return; } /* * Delete the entry from shm map. */ for (;;) { struct shmid_ds *shmseg; struct uvm_object *uobj; size_t sz; shmmap_se = SLIST_FIRST(&shmmap_s->entries); KASSERT(shmmap_se != NULL); shmseg = &shmsegs[IPCID_TO_IX(shmmap_se->shmid)]; sz = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; /* shm_delete_mapping() removes from the list. */ uobj = shm_delete_mapping(shmmap_s, shmmap_se); mutex_exit(&shm_lock); uvm_deallocate(&vm->vm_map, shmmap_se->va, sz); if (uobj != NULL) { uao_detach(uobj); } kmem_free(shmmap_se, sizeof(struct shmmap_entry)); if (SLIST_EMPTY(&shmmap_s->entries)) { break; } mutex_enter(&shm_lock); KASSERT(!SLIST_EMPTY(&shmmap_s->entries)); } kmem_free(shmmap_s, sizeof(struct shmmap_state)); } static int shmrealloc(int newshmni) { vaddr_t v; struct shmid_ds *oldshmsegs, *newshmsegs; kcondvar_t *newshm_cv, *oldshm_cv; size_t sz; int i, lsegid, oldshmni; if (newshmni < 1) return EINVAL; /* Allocate new memory area */ sz = ALIGN(newshmni * sizeof(struct shmid_ds)) + ALIGN(newshmni * sizeof(kcondvar_t)); sz = round_page(sz); v = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); if (v == 0) return ENOMEM; mutex_enter(&shm_lock); while (shm_realloc_state || shm_realloc_disable) cv_wait(&shm_realloc_cv, &shm_lock); /* * Get the number of last segment. Fail we are trying to * reallocate less memory than we use. */ lsegid = 0; for (i = 0; i < shminfo.shmmni; i++) if ((shmsegs[i].shm_perm.mode & SHMSEG_FREE) == 0) lsegid = i; if (lsegid >= newshmni) { mutex_exit(&shm_lock); uvm_km_free(kernel_map, v, sz, UVM_KMF_WIRED); return EBUSY; } shm_realloc_state = true; newshmsegs = (void *)v; newshm_cv = (void *)((uintptr_t)newshmsegs + ALIGN(newshmni * sizeof(struct shmid_ds))); /* Copy all memory to the new area */ for (i = 0; i < shm_nused; i++) { cv_init(&newshm_cv[i], "shmwait"); (void)memcpy(&newshmsegs[i], &shmsegs[i], sizeof(newshmsegs[0])); } /* Mark as free all new segments, if there is any */ for (; i < newshmni; i++) { cv_init(&newshm_cv[i], "shmwait"); newshmsegs[i].shm_perm.mode = SHMSEG_FREE; newshmsegs[i].shm_perm._seq = 0; } oldshmsegs = shmsegs; oldshmni = shminfo.shmmni; shminfo.shmmni = newshmni; shmsegs = newshmsegs; shm_cv = newshm_cv; /* Reallocation completed - notify all waiters, if any */ shm_realloc_state = false; cv_broadcast(&shm_realloc_cv); mutex_exit(&shm_lock); /* Release now unused resources. */ oldshm_cv = (void *)((uintptr_t)oldshmsegs + ALIGN(oldshmni * sizeof(struct shmid_ds))); for (i = 0; i < oldshmni; i++) cv_destroy(&oldshm_cv[i]); sz = ALIGN(oldshmni * sizeof(struct shmid_ds)) + ALIGN(oldshmni * sizeof(kcondvar_t)); sz = round_page(sz); uvm_km_free(kernel_map, (vaddr_t)oldshmsegs, sz, UVM_KMF_WIRED); return 0; } int shminit(struct sysctllog **clog) { vaddr_t v; size_t sz; int i; mutex_init(&shm_lock, MUTEX_DEFAULT, IPL_NONE); cv_init(&shm_realloc_cv, "shmrealc"); /* Allocate the wired memory for our structures */ sz = ALIGN(shminfo.shmmni * sizeof(struct shmid_ds)) + ALIGN(shminfo.shmmni * sizeof(kcondvar_t)); sz = round_page(sz); v = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); if (v == 0) { printf("sysv_shm: cannot allocate memory"); return ENOMEM; } shmsegs = (void *)v; shm_cv = (void *)((uintptr_t)shmsegs + ALIGN(shminfo.shmmni * sizeof(struct shmid_ds))); if (shminfo.shmmax == 0) shminfo.shmall = uimax(physmem / 4, 1024); else shminfo.shmall = shminfo.shmmax / PAGE_SIZE; shminfo.shmmax = (uint64_t)shminfo.shmall * PAGE_SIZE; for (i = 0; i < shminfo.shmmni; i++) { cv_init(&shm_cv[i], "shmwait"); shmsegs[i].shm_perm.mode = SHMSEG_FREE; shmsegs[i].shm_perm._seq = 0; } shm_last_free = 0; shm_nused = 0; shm_committed = 0; shm_realloc_disable = 0; shm_realloc_state = false; kern_has_sysvshm = 1; /* Load the callback function pointers for the uvm subsystem */ uvm_shmexit = shmexit; uvm_shmfork = shmfork; #ifdef _MODULE if (clog) sysctl_ipc_shm_setup(clog); #endif return 0; } int shmfini(void) { size_t sz; int i; vaddr_t v = (vaddr_t)shmsegs; mutex_enter(&shm_lock); if (shm_nused) { mutex_exit(&shm_lock); return 1; } /* Clear the callback function pointers for the uvm subsystem */ uvm_shmexit = NULL; uvm_shmfork = NULL; /* Destroy all condvars */ for (i = 0; i < shminfo.shmmni; i++) cv_destroy(&shm_cv[i]); cv_destroy(&shm_realloc_cv); /* Free the allocated/wired memory */ sz = ALIGN(shminfo.shmmni * sizeof(struct shmid_ds)) + ALIGN(shminfo.shmmni * sizeof(kcondvar_t)); sz = round_page(sz); uvm_km_free(kernel_map, v, sz, UVM_KMF_WIRED); /* Release and destroy our mutex */ mutex_exit(&shm_lock); mutex_destroy(&shm_lock); kern_has_sysvshm = 0; return 0; } static int sysctl_ipc_shmmni(SYSCTLFN_ARGS) { int newsize, error; struct sysctlnode node; node = *rnode; node.sysctl_data = &newsize; newsize = shminfo.shmmni; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; sysctl_unlock(); error = shmrealloc(newsize); sysctl_relock(); return error; } static int sysctl_ipc_shmmaxpgs(SYSCTLFN_ARGS) { uint32_t newsize; int error; struct sysctlnode node; node = *rnode; node.sysctl_data = &newsize; newsize = shminfo.shmall; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; if (newsize < 1) return EINVAL; shminfo.shmall = newsize; shminfo.shmmax = (uint64_t)shminfo.shmall * PAGE_SIZE; return 0; } static int sysctl_ipc_shmmax(SYSCTLFN_ARGS) { uint64_t newsize; int error; struct sysctlnode node; node = *rnode; node.sysctl_data = &newsize; newsize = shminfo.shmmax; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; if (newsize < PAGE_SIZE) return EINVAL; shminfo.shmmax = round_page(newsize); shminfo.shmall = shminfo.shmmax / PAGE_SIZE; return 0; } SYSCTL_SETUP(sysctl_ipc_shm_setup, "sysctl kern.ipc subtree setup") { sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT, CTLTYPE_NODE, "ipc", SYSCTL_DESCR("SysV IPC options"), NULL, 0, NULL, 0, CTL_KERN, KERN_SYSVIPC, CTL_EOL); sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_QUAD, "shmmax", SYSCTL_DESCR("Max shared memory segment size in bytes"), sysctl_ipc_shmmax, 0, &shminfo.shmmax, 0, CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMAX, CTL_EOL); sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_INT, "shmmni", SYSCTL_DESCR("Max number of shared memory identifiers"), sysctl_ipc_shmmni, 0, &shminfo.shmmni, 0, CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMNI, CTL_EOL); sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_INT, "shmseg", SYSCTL_DESCR("Max shared memory segments per process"), NULL, 0, &shminfo.shmseg, 0, CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMSEG, CTL_EOL); sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_INT, "shmmaxpgs", SYSCTL_DESCR("Max amount of shared memory in pages"), sysctl_ipc_shmmaxpgs, 0, &shminfo.shmall, 0, CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMAXPGS, CTL_EOL); sysctl_createv(clog, 0, NULL, NULL, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_INT, "shm_use_phys", SYSCTL_DESCR("Enable/disable locking of shared memory in " "physical memory"), NULL, 0, &shm_use_phys, 0, CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMUSEPHYS, CTL_EOL); }