/*-
* Copyright (c) 1998, 2000, 2004, 2008, 2009, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
* NASA Ames Research Center, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Copyright (c) 1982, 1986, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)uipc_usrreq.c 8.9 (Berkeley) 5/14/95
*/
/*
* Copyright (c) 1997 Christopher G. Demetriou. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)uipc_usrreq.c 8.9 (Berkeley) 5/14/95
*/
/*
* Unix communications domain.
*
* TODO:
* RDM
* rethink name space problems
* need a proper out-of-band
*
* Notes on locking:
*
* The generic rules noted in uipc_socket2.c apply. In addition:
*
* o We have a global lock, uipc_lock.
*
* o All datagram sockets are locked by uipc_lock.
*
* o For stream socketpairs, the two endpoints are created sharing the same
* independent lock. Sockets presented to PRU_CONNECT2 must already have
* matching locks.
*
* o Stream sockets created via socket() start life with their own
* independent lock.
*
* o Stream connections to a named endpoint are slightly more complicated.
* Sockets that have called listen() have their lock pointer mutated to
* the global uipc_lock. When establishing a connection, the connecting
* socket also has its lock mutated to uipc_lock, which matches the head
* (listening socket). We create a new socket for accept() to return, and
* that also shares the head's lock. Until the connection is completely
* done on both ends, all three sockets are locked by uipc_lock. Once the
* connection is complete, the association with the head's lock is broken.
* The connecting socket and the socket returned from accept() have their
* lock pointers mutated away from uipc_lock, and back to the connecting
* socket's original, independent lock. The head continues to be locked
* by uipc_lock.
*
* o If uipc_lock is determined to be a significant source of contention,
* it could easily be hashed out. It is difficult to simply make it an
* independent lock because of visibility / garbage collection issues:
* if a socket has been associated with a lock at any point, that lock
* must remain valid until the socket is no longer visible in the system.
* The lock must not be freed or otherwise destroyed until any sockets
* that had referenced it have also been destroyed.
*/
const struct sockaddr_un sun_noname = {
.sun_len = offsetof(struct sockaddr_un, sun_path),
.sun_family = AF_LOCAL,
};
ino_t unp_ino; /* prototype for fake inode numbers */
/*
* A connection succeeded: disassociate both endpoints from the head's
* lock, and make them share their own lock. There is a race here: for
* a very brief time one endpoint will be locked by a different lock
* than the other end. However, since the current thread holds the old
* lock (the listening socket's lock, the head) access can still only be
* made to one side of the connection.
*/
static void
unp_setpeerlocks(struct socket *so, struct socket *so2)
{
struct unpcb *unp;
kmutex_t *lock;
KASSERT(solocked2(so, so2));
/*
* Bail out if either end of the socket is not yet fully
* connected or accepted. We only break the lock association
* with the head when the pair of sockets stand completely
* on their own.
*/
KASSERT(so->so_head == NULL);
if (so2->so_head != NULL)
return;
/*
* Drop references to old lock. A third reference (from the
* queue head) must be held as we still hold its lock. Bonus:
* we don't need to worry about garbage collecting the lock.
*/
lock = so->so_lock;
KASSERT(lock == uipc_lock);
mutex_obj_free(lock);
mutex_obj_free(lock);
/*
* Grab stream lock from the initiator and share between the two
* endpoints. Issue memory barrier to ensure all modifications
* become globally visible before the lock change. so2 is
* assumed not to have a stream lock, because it was created
* purely for the server side to accept this connection and
* started out life using the domain-wide lock.
*/
unp = sotounpcb(so);
KASSERT(unp->unp_streamlock != NULL);
KASSERT(sotounpcb(so2)->unp_streamlock == NULL);
lock = unp->unp_streamlock;
unp->unp_streamlock = NULL;
mutex_obj_hold(lock);
/*
* Ensure lock is initialized before publishing it with
* solockreset. Pairs with atomic_load_consume in solock and
* various loops to reacquire lock after wakeup.
*/
membar_release();
/*
* possible race if lock is not held - see comment in
* uipc_usrreq(PRU_ACCEPT).
*/
KASSERT(mutex_owned(lock));
solockreset(so, lock);
solockreset(so2, lock);
}
/*
* Reset a socket's lock back to the domain-wide lock.
*/
static void
unp_resetlock(struct socket *so)
{
kmutex_t *olock, *nlock;
struct unpcb *unp;
if (peeraddr) {
if (unp->unp_conn && unp->unp_conn->unp_addr)
sun = unp->unp_conn->unp_addr;
} else {
if (unp->unp_addr)
sun = unp->unp_addr;
}
if (sun == NULL)
sun = &sun_noname;
/*
* Note: unp_internalize() rejects any control message
* other than SCM_RIGHTS, and only allows one. This
* has the side-effect of preventing a caller from
* forging SCM_CREDS.
*/
if (control) {
sounlock(so);
error = unp_internalize(&control);
solock(so);
if (error != 0) {
m_freem(control);
m_freem(m);
return error;
}
}
switch (so->so_type) {
case SOCK_DGRAM: {
KASSERT(so->so_lock == uipc_lock);
if (nam) {
if ((so->so_state & SS_ISCONNECTED) != 0)
error = SET_ERROR(EISCONN);
else {
/*
* Note: once connected, the
* socket's lock must not be
* dropped until we have sent
* the message and disconnected.
* This is necessary to prevent
* intervening control ops, like
* another connection.
*/
error = unp_connect(so, nam, l);
}
} else {
if ((so->so_state & SS_ISCONNECTED) == 0)
error = SET_ERROR(ENOTCONN);
}
if (error) {
unp_dispose(control);
m_freem(control);
m_freem(m);
return error;
}
error = unp_output(m, control, unp);
if (nam)
unp_disconnect1(unp);
break;
}
case SOCK_SEQPACKET: /* FALLTHROUGH */
case SOCK_STREAM:
#define rcv (&so2->so_rcv)
#define snd (&so->so_snd)
if (unp->unp_conn == NULL) {
error = SET_ERROR(ENOTCONN);
break;
}
so2 = unp->unp_conn->unp_socket;
KASSERT(solocked2(so, so2));
if (unp->unp_conn->unp_flags & UNP_WANTCRED) {
/*
* Credentials are passed only once on
* SOCK_STREAM and SOCK_SEQPACKET.
*/
unp->unp_conn->unp_flags &= ~UNP_WANTCRED;
control = unp_addsockcred(l, control);
}
if (unp->unp_conn->unp_flags & UNP_OWANTCRED) {
/*
* Credentials are passed only once on
* SOCK_STREAM and SOCK_SEQPACKET.
*/
unp->unp_conn->unp_flags &= ~UNP_OWANTCRED;
MODULE_HOOK_CALL(uipc_unp_70_hook, (curlwp, control),
stub_compat_70_unp_addsockcred(curlwp, control),
control);
}
/*
* Send to paired receive port, and then reduce
* send buffer hiwater marks to maintain backpressure.
* Wake up readers.
*/
if (control) {
if (sbappendcontrol(rcv, m, control) != 0)
control = NULL;
} else {
switch(so->so_type) {
case SOCK_SEQPACKET:
sbappendrecord(rcv, m);
break;
case SOCK_STREAM:
sbappend(rcv, m);
break;
default:
panic("uipc_usrreq");
break;
}
}
snd->sb_mbmax -=
rcv->sb_mbcnt - unp->unp_conn->unp_mbcnt;
unp->unp_conn->unp_mbcnt = rcv->sb_mbcnt;
newhiwat = snd->sb_hiwat -
(rcv->sb_cc - unp->unp_conn->unp_cc);
(void)chgsbsize(so->so_uidinfo,
&snd->sb_hiwat, newhiwat, RLIM_INFINITY);
unp->unp_conn->unp_cc = rcv->sb_cc;
sorwakeup(so2);
#undef snd
#undef rcv
if (control != NULL) {
unp_dispose(control);
m_freem(control);
}
break;
/*
* Both send and receive buffers are allocated PIPSIZ bytes of buffering
* for stream sockets, although the total for sender and receiver is
* actually only PIPSIZ.
* Datagram sockets really use the sendspace as the maximum datagram size,
* and don't really want to reserve the sendspace. Their recvspace should
* be large enough for at least one max-size datagram plus address.
*/
#ifndef PIPSIZ
#define PIPSIZ 8192
#endif
u_long unpst_sendspace = PIPSIZ;
u_long unpst_recvspace = PIPSIZ;
u_long unpdg_sendspace = 2*1024; /* really max datagram size */
u_long unpdg_recvspace = 16*1024;
u_int unp_rights; /* files in flight */
u_int unp_rights_ratio = 2; /* limit, fraction of maxfiles */
static int
unp_attach(struct socket *so, int proto)
{
struct unpcb *unp = sotounpcb(so);
u_long sndspc, rcvspc;
int error;
KASSERT(unp == NULL);
switch (so->so_type) {
case SOCK_SEQPACKET:
/* FALLTHROUGH */
case SOCK_STREAM:
if (so->so_lock == NULL) {
so->so_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
solock(so);
}
sndspc = unpst_sendspace;
rcvspc = unpst_recvspace;
break;
case SOCK_DGRAM:
if (so->so_lock == NULL) {
mutex_obj_hold(uipc_lock);
so->so_lock = uipc_lock;
solock(so);
}
sndspc = unpdg_sendspace;
rcvspc = unpdg_recvspace;
break;
unp = sotounpcb(so);
KASSERT(unp != NULL);
KASSERT(solocked(so));
retry:
if ((vp = unp->unp_vnode) != NULL) {
sounlock(so);
/* Acquire v_interlock to protect against unp_connect(). */
/* XXXAD racy */
mutex_enter(vp->v_interlock);
vp->v_socket = NULL;
mutex_exit(vp->v_interlock);
vrele(vp);
solock(so);
unp->unp_vnode = NULL;
}
if (unp->unp_conn)
unp_disconnect1(unp);
while (unp->unp_refs) {
KASSERT(solocked2(so, unp->unp_refs->unp_socket));
if (unp_drop(unp->unp_refs, SET_ERROR(ECONNRESET))) {
solock(so);
goto retry;
}
}
soisdisconnected(so);
so->so_pcb = NULL;
if (unp_rights) {
/*
* Normally the receive buffer is flushed later, in sofree,
* but if our receive buffer holds references to files that
* are now garbage, we will enqueue those file references to
* the garbage collector and kick it into action.
*/
sorflush(so);
unp_free(unp);
unp_thread_kick();
} else
unp_free(unp);
}
/* XXX code review required to determine if unp can ever be NULL */
if (unp == NULL)
return SET_ERROR(EINVAL);
KASSERT(so->so_lock == uipc_lock);
/*
* Mark the initiating STREAM socket as connected *ONLY*
* after it's been accepted. This prevents a client from
* overrunning a server and receiving ECONNREFUSED.
*/
if (unp->unp_conn == NULL) {
/*
* This will use the empty socket and will not
* allocate.
*/
unp_setaddr(so, nam, true);
return 0;
}
so2 = unp->unp_conn->unp_socket;
if (so2->so_state & SS_ISCONNECTING) {
KASSERT(so->so_head == NULL || solocked2(so, so->so_head));
KASSERT(so->so_head == NULL || solocked2(so2, so->so_head));
soisconnected(so2);
}
/*
* If the connection is fully established, break the
* association with uipc_lock and give the connected
* pair a separate lock to share.
* There is a race here: sotounpcb(so2)->unp_streamlock
* is not locked, so when changing so2->so_lock
* another thread can grab it while so->so_lock is still
* pointing to the (locked) uipc_lock.
* this should be harmless, except that this makes
* solocked2() and solocked() unreliable.
* Another problem is that unp_setaddr() expects the
* the socket locked. Grabbing sotounpcb(so2)->unp_streamlock
* fixes both issues.
*/
mutex_enter(sotounpcb(so2)->unp_streamlock);
unp_setpeerlocks(so2, so);
/*
* Only now return peer's address, as we may need to
* block in order to allocate memory.
*
* XXX Minor race: connection can be broken while
* lock is dropped in unp_setaddr(). We will return
* error == 0 and sun_noname as the peer address.
*/
unp_setaddr(so, nam, true);
/* so_lock now points to unp_streamlock */
mutex_exit(so2->so_lock);
return 0;
}
/*
* we only need to perform this allocation until syscalls other than
* bind are adjusted to use sockaddr_big.
*/
static struct sockaddr_un *
makeun_sb(struct sockaddr *nam, size_t *addrlen)
{
struct sockaddr_un *sun;
if (unp->unp_vnode != NULL)
return SET_ERROR(EINVAL);
if ((unp->unp_flags & UNP_BUSY) != 0) {
/*
* EALREADY may not be strictly accurate, but since this
* is a major application error it's hardly a big deal.
*/
return SET_ERROR(EALREADY);
}
unp->unp_flags |= UNP_BUSY;
sounlock(so);
/*
* If the socket can accept a connection, it must be
* locked by uipc_lock.
*/
unp_resetlock(so);
if (unp->unp_vnode == NULL)
return SET_ERROR(EINVAL);
unp_connid(l, unp, UNP_EIDSBIND);
return 0;
}
static int
unp_disconnect(struct socket *so)
{
KASSERT(solocked(so));
KASSERT(sotounpcb(so) != NULL);
unp_disconnect1(sotounpcb(so));
return 0;
}
static int
unp_shutdown(struct socket *so)
{
KASSERT(solocked(so));
KASSERT(sotounpcb(so) != NULL);
if (so2->so_type != so->so_type)
return SET_ERROR(EPROTOTYPE);
/*
* All three sockets involved must be locked by same lock:
*
* local endpoint (so)
* remote endpoint (so2)
* queue head (so2->so_head, only if PR_CONNREQUIRED)
*/
KASSERT(solocked2(so, so2));
KASSERT(so->so_head == NULL);
if (so2->so_head != NULL) {
KASSERT(so2->so_lock == uipc_lock);
KASSERT(solocked2(so2, so2->so_head));
}
unp2 = sotounpcb(so2);
unp->unp_conn = unp2;
switch (so->so_type) {
case SOCK_DGRAM:
unp->unp_nextref = unp2->unp_refs;
unp2->unp_refs = unp;
soisconnected(so);
break;
case SOCK_SEQPACKET: /* FALLTHROUGH */
case SOCK_STREAM:
/*
* SOCK_SEQPACKET and SOCK_STREAM cases are handled by callers
* which are unp_connect() or unp_connect2().
*/
unp = sotounpcb(so);
if ((unp->unp_flags & UNP_BUSY) != 0) {
/*
* EALREADY may not be strictly accurate, but since this
* is a major application error it's hardly a big deal.
*/
return SET_ERROR(EALREADY);
}
unp->unp_flags |= UNP_BUSY;
sounlock(so);
sun = makeun_sb(nam, &addrlen);
pb = pathbuf_create(sun->sun_path);
if (pb == NULL) {
error = SET_ERROR(ENOMEM);
goto bad2;
}
if ((error = namei(&nd)) != 0) {
pathbuf_destroy(pb);
goto bad2;
}
vp = nd.ni_vp;
pathbuf_destroy(pb);
if (vp->v_type != VSOCK) {
error = SET_ERROR(ENOTSOCK);
goto bad;
}
if ((error = VOP_ACCESS(vp, VWRITE, l->l_cred)) != 0)
goto bad;
/* Acquire v_interlock to protect against unp_detach(). */
mutex_enter(vp->v_interlock);
so2 = vp->v_socket;
if (so2 == NULL) {
mutex_exit(vp->v_interlock);
error = SET_ERROR(ECONNREFUSED);
goto bad;
}
if (so->so_type != so2->so_type) {
mutex_exit(vp->v_interlock);
error = SET_ERROR(EPROTOTYPE);
goto bad;
}
solock(so);
unp_resetlock(so);
mutex_exit(vp->v_interlock);
if ((so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
/*
* This may seem somewhat fragile but is OK: if we can
* see SO_ACCEPTCONN set on the endpoint, then it must
* be locked by the domain-wide uipc_lock.
*/
KASSERT((so2->so_options & SO_ACCEPTCONN) == 0 ||
so2->so_lock == uipc_lock);
if ((so2->so_options & SO_ACCEPTCONN) == 0 ||
(so3 = sonewconn(so2, false)) == NULL) {
error = SET_ERROR(ECONNREFUSED);
sounlock(so);
goto bad;
}
unp2 = sotounpcb(so2);
unp3 = sotounpcb(so3);
if (unp2->unp_addr) {
unp3->unp_addr = malloc(unp2->unp_addrlen,
M_SONAME, M_WAITOK);
memcpy(unp3->unp_addr, unp2->unp_addr,
unp2->unp_addrlen);
unp3->unp_addrlen = unp2->unp_addrlen;
}
unp3->unp_flags = unp2->unp_flags;
so2 = so3;
/*
* The connector's (client's) credentials are copied from its
* process structure at the time of connect() (which is now).
*/
unp_connid(l, unp3, UNP_EIDSVALID);
/*
* The receiver's (server's) credentials are copied from the
* unp_peercred member of socket on which the former called
* listen(); unp_listen() cached that process's credentials
* at that time so we can use them now.
*/
if (unp2->unp_flags & UNP_EIDSBIND) {
memcpy(&unp->unp_connid, &unp2->unp_connid,
sizeof(unp->unp_connid));
unp->unp_flags |= UNP_EIDSVALID;
}
}
error = unp_connect1(so, so2, l);
if (error) {
sounlock(so);
goto bad;
}
unp2 = sotounpcb(so2);
switch (so->so_type) {
/*
* SOCK_DGRAM and default cases are handled in prior call to
* unp_connect1(), do not add a default case without fixing
* unp_connect1().
*/
case SOCK_SEQPACKET: /* FALLTHROUGH */
case SOCK_STREAM:
unp2->unp_conn = unp;
if ((unp->unp_flags | unp2->unp_flags) & UNP_CONNWAIT)
soisconnecting(so);
else
soisconnected(so);
soisconnected(so2);
/*
* If the connection is fully established, break the
* association with uipc_lock and give the connected
* pair a separate lock to share.
*/
KASSERT(so2->so_head != NULL);
unp_setpeerlocks(so, so2);
break;
int * const fdp = kmem_alloc(nfds * sizeof(int), KM_SLEEP);
rw_enter(&p->p_cwdi->cwdi_lock, RW_READER);
/* Make sure the recipient should be able to see the files.. */
rp = (file_t **)CMSG_DATA(cm);
for (size_t i = 0; i < nfds; i++) {
file_t * const fp = *rp++;
if (fp == NULL) {
error = SET_ERROR(EINVAL);
goto out;
}
/*
* If we are in a chroot'ed directory, and
* someone wants to pass us a directory, make
* sure it's inside the subtree we're allowed
* to access.
*/
if (p->p_cwdi->cwdi_rdir != NULL && fp->f_type == DTYPE_VNODE) {
vnode_t *vp = fp->f_vnode;
if ((vp->v_type == VDIR) &&
!vn_isunder(vp, p->p_cwdi->cwdi_rdir, l)) {
error = SET_ERROR(EPERM);
goto out;
}
}
}
restart:
/*
* First loop -- allocate file descriptor table slots for the
* new files.
*/
for (size_t i = 0; i < nfds; i++) {
if ((error = fd_alloc(p, 0, &fdp[i])) != 0) {
/*
* Back out what we've done so far.
*/
while (i-- > 0) {
fd_abort(p, NULL, fdp[i]);
}
if (error == ENOSPC) {
fd_tryexpand(p);
error = 0;
goto restart;
}
/*
* This is the error that has historically
* been returned, and some callers may
* expect it.
*/
error = SET_ERROR(EMSGSIZE);
goto out;
}
}
/*
* Now that adding them has succeeded, update all of the
* file passing state and affix the descriptors.
*/
rp = (file_t **)CMSG_DATA(cm);
int *ofdp = (int *)CMSG_DATA(cm);
for (size_t i = 0; i < nfds; i++) {
file_t * const fp = *rp++;
const int fd = fdp[i];
atomic_dec_uint(&unp_rights);
fd_set_exclose(l, fd, (flags & O_CLOEXEC) != 0);
fd_affix(p, fp, fd);
/*
* Done with this file pointer, replace it with a fd;
*/
*ofdp++ = fd;
mutex_enter(&fp->f_lock);
fp->f_msgcount--;
mutex_exit(&fp->f_lock);
/*
* Note that fd_affix() adds a reference to the file.
* The file may already have been closed by another
* LWP in the process, so we must drop the reference
* added by unp_internalize() with closef().
*/
closef(fp);
}
/*
* Adjust length, in case of transition from large file_t
* pointers to ints.
*/
if (sizeof(file_t *) != sizeof(int)) {
cm->cmsg_len = CMSG_LEN(nfds * sizeof(int));
rights->m_len = CMSG_SPACE(nfds * sizeof(int));
}
out:
if (__predict_false(error != 0)) {
file_t **const fpp = (file_t **)CMSG_DATA(cm);
for (size_t i = 0; i < nfds; i++)
unp_discard_now(fpp[i]);
/*
* Truncate the array so that nobody will try to interpret
* what is now garbage in it.
*/
cm->cmsg_len = CMSG_LEN(0);
rights->m_len = CMSG_SPACE(0);
}
rw_exit(&p->p_cwdi->cwdi_lock);
kmem_free(fdp, nfds * sizeof(int));
static int
unp_internalize(struct mbuf **controlp)
{
filedesc_t *fdescp = curlwp->l_fd;
fdtab_t *dt;
struct mbuf *control = *controlp;
struct cmsghdr *newcm, *cm = mtod(control, struct cmsghdr *);
file_t **rp, **files;
file_t *fp;
int i, fd, *fdp;
int nfds, error;
u_int maxmsg;
error = 0;
newcm = NULL;
/* Sanity check the control message header. */
if (cm->cmsg_type != SCM_RIGHTS || cm->cmsg_level != SOL_SOCKET ||
cm->cmsg_len > control->m_len ||
cm->cmsg_len < CMSG_ALIGN(sizeof(*cm)))
return SET_ERROR(EINVAL);
/*
* Verify that the file descriptors are valid, and acquire
* a reference to each.
*/
nfds = (cm->cmsg_len - CMSG_ALIGN(sizeof(*cm))) / sizeof(int);
fdp = (int *)CMSG_DATA(cm);
maxmsg = maxfiles / unp_rights_ratio;
for (i = 0; i < nfds; i++) {
fd = *fdp++;
if (atomic_inc_uint_nv(&unp_rights) > maxmsg) {
atomic_dec_uint(&unp_rights);
nfds = i;
error = SET_ERROR(EAGAIN);
goto out;
}
if ((fp = fd_getfile(fd)) == NULL
|| fp->f_type == DTYPE_KQUEUE) {
if (fp)
fd_putfile(fd);
atomic_dec_uint(&unp_rights);
nfds = i;
error = SET_ERROR(EBADF);
goto out;
}
}
/* Allocate new space and copy header into it. */
newcm = malloc(CMSG_SPACE(nfds * sizeof(file_t *)), M_MBUF, M_WAITOK);
if (newcm == NULL) {
error = SET_ERROR(E2BIG);
goto out;
}
memcpy(newcm, cm, sizeof(struct cmsghdr));
memset(newcm + 1, 0, CMSG_LEN(0) - sizeof(struct cmsghdr));
files = (file_t **)CMSG_DATA(newcm);
/*
* Transform the file descriptors into file_t pointers, in
* reverse order so that if pointers are bigger than ints, the
* int won't get until we're done. No need to lock, as we have
* already validated the descriptors with fd_getfile().
*/
fdp = (int *)CMSG_DATA(cm) + nfds;
rp = files + nfds;
for (i = 0; i < nfds; i++) {
dt = atomic_load_consume(&fdescp->fd_dt);
fp = atomic_load_consume(&dt->dt_ff[*--fdp]->ff_file);
KASSERT(fp != NULL);
mutex_enter(&fp->f_lock);
*--rp = fp;
fp->f_count++;
fp->f_msgcount++;
mutex_exit(&fp->f_lock);
}
out:
/* Release descriptor references. */
fdp = (int *)CMSG_DATA(cm);
for (i = 0; i < nfds; i++) {
fd_putfile(*fdp++);
if (error != 0) {
atomic_dec_uint(&unp_rights);
}
}
if (error == 0) {
if (control->m_flags & M_EXT) {
m_freem(control);
*controlp = control = m_get(M_WAIT, MT_CONTROL);
}
MEXTADD(control, newcm, CMSG_SPACE(nfds * sizeof(file_t *)),
M_MBUF, NULL, NULL);
cm = newcm;
/*
* Adjust message & mbuf to note amount of space
* actually used.
*/
cm->cmsg_len = CMSG_LEN(nfds * sizeof(file_t *));
control->m_len = CMSG_SPACE(nfds * sizeof(file_t *));
}
for (int i = 0; i < sc->sc_ngroups; i++)
sc->sc_groups[i] = kauth_cred_group(l->l_cred, i);
return m_add(control, m);
}
/*
* Do a mark-sweep GC of files in the system, to free up any which are
* caught in flight to an about-to-be-closed socket. Additionally,
* process deferred file closures.
*/
static void
unp_gc(file_t *dp)
{
extern struct domain unixdomain;
file_t *fp, *np;
struct socket *so, *so1;
u_int i, oflags, rflags;
bool didwork;
/*
* Clear mark bits. Ensure that we don't consider new files
* entering the file table during this loop (they will not have
* FSCAN set).
*/
unp_defer = 0;
LIST_FOREACH(fp, &filehead, f_list) {
for (oflags = fp->f_flag;; oflags = rflags) {
rflags = atomic_cas_uint(&fp->f_flag, oflags,
(oflags | FSCAN) & ~(FMARK|FDEFER));
if (__predict_true(oflags == rflags)) {
break;
}
}
}
/*
* Iterate over the set of sockets, marking ones believed (based on
* refcount) to be referenced from a process, and marking for rescan
* sockets which are queued on a socket. Recan continues descending
* and searching for sockets referenced by sockets (FDEFER), until
* there are no more socket->socket references to be discovered.
*/
do {
didwork = false;
for (fp = LIST_FIRST(&filehead); fp != NULL; fp = np) {
KASSERT(mutex_owned(&filelist_lock));
np = LIST_NEXT(fp, f_list);
mutex_enter(&fp->f_lock);
if ((fp->f_flag & FDEFER) != 0) {
atomic_and_uint(&fp->f_flag, ~FDEFER);
unp_defer--;
if (fp->f_count == 0) {
/*
* XXX: closef() doesn't pay attention
* to FDEFER
*/
mutex_exit(&fp->f_lock);
continue;
}
} else {
if (fp->f_count == 0 ||
(fp->f_flag & FMARK) != 0 ||
fp->f_count == fp->f_msgcount ||
fp->f_unpcount != 0) {
mutex_exit(&fp->f_lock);
continue;
}
}
atomic_or_uint(&fp->f_flag, FMARK);
/* Gain file ref, mark our position, and unlock. */
didwork = true;
LIST_INSERT_AFTER(fp, dp, f_list);
fp->f_count++;
mutex_exit(&fp->f_lock);
mutex_exit(&filelist_lock);
/*
* Mark files referenced from sockets queued on the
* accept queue as well.
*/
solock(so);
unp_scan(so->so_rcv.sb_mb, unp_mark, 0);
if ((so->so_options & SO_ACCEPTCONN) != 0) {
TAILQ_FOREACH(so1, &so->so_q0, so_qe) {
unp_scan(so1->so_rcv.sb_mb, unp_mark, 0);
}
TAILQ_FOREACH(so1, &so->so_q, so_qe) {
unp_scan(so1->so_rcv.sb_mb, unp_mark, 0);
}
}
sounlock(so);
/* Re-lock and restart from where we left off. */
closef(fp);
mutex_enter(&filelist_lock);
np = LIST_NEXT(dp, f_list);
LIST_REMOVE(dp, f_list);
}
/*
* Bail early if we did nothing in the loop above. Could
* happen because of concurrent activity causing unp_defer
* to get out of sync.
*/
} while (unp_defer != 0 && didwork);
/*
* Sweep pass.
*
* We grab an extra reference to each of the files that are
* not otherwise accessible and then free the rights that are
* stored in messages on them.
*/
for (fp = LIST_FIRST(&filehead); fp != NULL; fp = np) {
KASSERT(mutex_owned(&filelist_lock));
np = LIST_NEXT(fp, f_list);
mutex_enter(&fp->f_lock);
/*
* Ignore non-sockets.
* Ignore dead sockets, or sockets with pending close.
* Ignore sockets obviously referenced elsewhere.
* Ignore sockets marked as referenced by our scan.
* Ignore new sockets that did not exist during the scan.
*/
if (fp->f_type != DTYPE_SOCKET ||
fp->f_count == 0 || fp->f_unpcount != 0 ||
fp->f_count != fp->f_msgcount ||
(fp->f_flag & (FMARK | FSCAN)) != FSCAN) {
mutex_exit(&fp->f_lock);
continue;
}
/* Gain file ref, mark our position, and unlock. */
LIST_INSERT_AFTER(fp, dp, f_list);
fp->f_count++;
mutex_exit(&fp->f_lock);
mutex_exit(&filelist_lock);
/*
* Flush all data from the socket's receive buffer.
* This will cause files referenced only by the
* socket to be queued for close.
*/
so = fp->f_socket;
solock(so);
sorflush(so);
sounlock(so);
/* Re-lock and restart from where we left off. */
closef(fp);
mutex_enter(&filelist_lock);
np = LIST_NEXT(dp, f_list);
LIST_REMOVE(dp, f_list);
}
}
/*
* Garbage collector thread. While SCM_RIGHTS messages are in transit,
* wake once per second to garbage collect. Run continually while we
* have deferred closes to process.
*/
static void
unp_thread(void *cookie)
{
file_t *dp;
/* Allocate a dummy file for our scans. */
if ((dp = fgetdummy()) == NULL) {
panic("unp_thread");
}
mutex_enter(&filelist_lock);
for (;;) {
KASSERT(mutex_owned(&filelist_lock));
if (SLIST_EMPTY(&unp_thread_discard)) {
if (unp_rights != 0) {
(void)cv_timedwait(&unp_thread_cv,
&filelist_lock, hz);
} else {
cv_wait(&unp_thread_cv, &filelist_lock);
}
}
unp_gc(dp);
}
/* NOTREACHED */
}
/*
* Kick the garbage collector into action if there is something for
* it to process.
*/
static void
unp_thread_kick(void)
{
void
unp_scan(struct mbuf *m0, void (*op)(file_t *), int discard)
{
struct mbuf *m;
file_t **rp, *fp;
struct cmsghdr *cm;
int i, qfds;
while (m0) {
for (m = m0; m; m = m->m_next) {
if (m->m_type != MT_CONTROL ||
m->m_len < sizeof(*cm)) {
continue;
}
cm = mtod(m, struct cmsghdr *);
if (cm->cmsg_level != SOL_SOCKET ||
cm->cmsg_type != SCM_RIGHTS)
continue;
qfds = (cm->cmsg_len - CMSG_ALIGN(sizeof(*cm)))
/ sizeof(file_t *);
rp = (file_t **)CMSG_DATA(cm);
for (i = 0; i < qfds; i++) {
fp = *rp;
if (discard) {
*rp = 0;
}
(*op)(fp);
rp++;
}
}
m0 = m0->m_nextpkt;
}
}
void
unp_mark(file_t *fp)
{
if (fp == NULL)
return;
/* If we're already deferred, don't screw up the defer count */
mutex_enter(&fp->f_lock);
if (fp->f_flag & (FMARK | FDEFER)) {
mutex_exit(&fp->f_lock);
return;
}
/*
* Minimize the number of deferrals... Sockets are the only type of
* file which can hold references to another file, so just mark
* other files, and defer unmarked sockets for the next pass.
*/
if (fp->f_type == DTYPE_SOCKET) {
unp_defer++;
KASSERT(fp->f_count != 0);
atomic_or_uint(&fp->f_flag, FDEFER);
} else {
atomic_or_uint(&fp->f_flag, FMARK);
}
mutex_exit(&fp->f_lock);
}
