/* $NetBSD: ipsec.c,v 1.179 2024/05/13 00:12:33 msaitoh Exp $ */
/* $FreeBSD: ipsec.c,v 1.2.2.2 2003/07/01 01:38:13 sam Exp $ */
/* $KAME: ipsec.c,v 1.103 2001/05/24 07:14:18 sakane Exp $ */

/*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*      notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*      notice, this list of conditions and the following disclaimer in the
*      documentation and/or other materials provided with the distribution.
* 3. Neither the name of the project nor the names of its contributors
*      may be used to endorse or promote products derived from this software
*      without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ipsec.c,v 1.179 2024/05/13 00:12:33 msaitoh Exp $");

/*
* IPsec controller part.
*/

#if defined(_KERNEL_OPT)
#include "opt_inet.h"
#include "opt_ipsec.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/domain.h>
#include <sys/protosw.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/errno.h>
#include <sys/time.h>
#include <sys/kernel.h>
#include <sys/syslog.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/kauth.h>
#include <sys/cpu.h>
#include <sys/kmem.h>
#include <sys/pserialize.h>

#include <net/if.h>
#include <net/route.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/in_var.h>
#include <netinet/udp.h>
#include <netinet/udp_var.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <netinet/ip_icmp.h>
#include <netinet/ip_private.h>

#include <netinet/ip6.h>
#ifdef INET6
#include <netinet6/ip6_var.h>
#endif
#include <netinet/in_pcb.h>
#include <netinet/in_offload.h>
#ifdef INET6
#include <netinet6/in6_pcb.h>
#include <netinet/icmp6.h>
#endif

#include <netipsec/ipsec.h>
#include <netipsec/ipsec_var.h>
#include <netipsec/ipsec_private.h>
#ifdef INET6
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/ah_var.h>
#include <netipsec/esp_var.h>
#include <netipsec/ipcomp.h>            /*XXX*/
#include <netipsec/ipcomp_var.h>

#include <netipsec/key.h>
#include <netipsec/keydb.h>
#include <netipsec/key_debug.h>

#include <netipsec/xform.h>

int ipsec_used = 0;
int ipsec_enabled = 1;

#ifdef IPSEC_DEBUG
int ipsec_debug = 1;

/*
* When set to 1, IPsec will send packets with the same sequence number.
* This allows to verify if the other side has proper replay attacks detection.
*/
int ipsec_replay = 0;

/*
* When set 1, IPsec will send packets with corrupted HMAC.
* This allows to verify if the other side properly detects modified packets.
*/
int ipsec_integrity = 0;
#else
int ipsec_debug = 0;
#endif

percpu_t *ipsecstat_percpu;

int ip4_ah_offsetmask = 0;      /* maybe IP_DF? */
int ip4_ipsec_dfbit = 2;        /* DF bit on encap. 0: clear 1: set 2: copy */
int ip4_esp_trans_deflev = IPSEC_LEVEL_USE;
int ip4_esp_net_deflev = IPSEC_LEVEL_USE;
int ip4_ah_trans_deflev = IPSEC_LEVEL_USE;
int ip4_ah_net_deflev = IPSEC_LEVEL_USE;
struct secpolicy ip4_def_policy;
int ip4_ipsec_ecn = 0;          /* ECN ignore(-1)/forbidden(0)/allowed(1) */

u_int ipsec_spdgen = 1;         /* SPD generation # */

static struct secpolicy ipsec_dummy_sp __read_mostly = {
       .state          = IPSEC_SPSTATE_ALIVE,
       /* If ENTRUST, the dummy SP never be used. See ipsec_getpolicybysock. */
       .policy         = IPSEC_POLICY_ENTRUST,
};

static struct secpolicy *ipsec_checkpcbcache(struct mbuf *,
   struct inpcbpolicy *, int);
static int ipsec_fillpcbcache(struct inpcbpolicy *, struct mbuf *,
   struct secpolicy *, int);
static int ipsec_invalpcbcache(struct inpcbpolicy *, int);

/*
* Crypto support requirements:
*
*  1   require hardware support
* -1   require software support
*  0   take anything
*/
int crypto_support = 0;

static struct secpolicy *ipsec_getpolicybysock(struct mbuf *, u_int,
   struct inpcb *, int *);

#ifdef INET6
int ip6_esp_trans_deflev = IPSEC_LEVEL_USE;
int ip6_esp_net_deflev = IPSEC_LEVEL_USE;
int ip6_ah_trans_deflev = IPSEC_LEVEL_USE;
int ip6_ah_net_deflev = IPSEC_LEVEL_USE;
struct secpolicy ip6_def_policy;
int ip6_ipsec_ecn = 0;          /* ECN ignore(-1)/forbidden(0)/allowed(1) */
#endif

static int ipsec_setspidx_inpcb(struct mbuf *, struct inpcb *);
static int ipsec_setspidx(struct mbuf *, struct secpolicyindex *, int, int);
static void ipsec4_get_ulp(struct mbuf *m, struct secpolicyindex *, int);
static int ipsec4_setspidx_ipaddr(struct mbuf *, struct secpolicyindex *);
#ifdef INET6
static void ipsec6_get_ulp(struct mbuf *m, struct secpolicyindex *, int);
static int ipsec6_setspidx_ipaddr(struct mbuf *, struct secpolicyindex *);
#endif
static void ipsec_delpcbpolicy(struct inpcbpolicy *);
static void ipsec_destroy_policy(struct secpolicy *);
static int ipsec_sp_reject(const struct secpolicy *, const struct mbuf *);
static void vshiftl(unsigned char *, int, int);
static size_t ipsec_sp_hdrsiz(const struct secpolicy *, const struct mbuf *);

/*
* Try to validate and use cached policy on a PCB.
*/
static struct secpolicy *
ipsec_checkpcbcache(struct mbuf *m, struct inpcbpolicy *pcbsp, int dir)
{
       struct secpolicyindex spidx;
       struct secpolicy *sp = NULL;
       int s;

       KASSERT(IPSEC_DIR_IS_VALID(dir));
       KASSERT(pcbsp != NULL);
       KASSERT(dir < __arraycount(pcbsp->sp_cache));
       KASSERT(inp_locked(pcbsp->sp_inp));

       /*
        * Checking the generation and sp->state and taking a reference to an SP
        * must be in a critical section of pserialize. See key_unlink_sp.
        */
       s = pserialize_read_enter();
       /* SPD table change invalidate all the caches. */
       if (ipsec_spdgen != pcbsp->sp_cache[dir].cachegen) {
               ipsec_invalpcbcache(pcbsp, dir);
               goto out;
       }
       sp = pcbsp->sp_cache[dir].cachesp;
       if (sp == NULL)
               goto out;
       if (sp->state != IPSEC_SPSTATE_ALIVE) {
               sp = NULL;
               ipsec_invalpcbcache(pcbsp, dir);
               goto out;
       }
       if ((pcbsp->sp_cacheflags & IPSEC_PCBSP_CONNECTED) == 0) {
               /* NB: assume ipsec_setspidx never sleep */
               if (ipsec_setspidx(m, &spidx, dir, 1) != 0) {
                       sp = NULL;
                       goto out;
               }

               /*
                * We have to make an exact match here since the cached rule
                * might have lower priority than a rule that would otherwise
                * have matched the packet.
                */
               if (memcmp(&pcbsp->sp_cache[dir].cacheidx, &spidx,
                   sizeof(spidx))) {
                       sp = NULL;
                       goto out;
               }
       } else {
               /*
                * The pcb is connected, and the L4 code is sure that:
                * - outgoing side uses inp_[lf]addr
                * - incoming side looks up policy after inpcb lookup
                * and address pair is know to be stable.  We do not need
                * to generate spidx again, nor check the address match again.
                *
                * For IPv4/v6 SOCK_STREAM sockets, this assumptions holds
                * and there are calls to ipsec_pcbconn() from inpcb_connect().
                */
       }

       key_sp_touch(sp);
       KEY_SP_REF(sp);
       KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_STAMP,
           "DP cause refcnt++:%d SP:%p\n",
           key_sp_refcnt(sp), pcbsp->sp_cache[dir].cachesp);
out:
       pserialize_read_exit(s);
       return sp;
}

static int
ipsec_fillpcbcache(struct inpcbpolicy *pcbsp, struct mbuf *m,
   struct secpolicy *sp, int dir)
{

       KASSERT(IPSEC_DIR_IS_INOROUT(dir));
       KASSERT(dir < __arraycount(pcbsp->sp_cache));
       KASSERT(inp_locked(pcbsp->sp_inp));

       pcbsp->sp_cache[dir].cachesp = NULL;
       pcbsp->sp_cache[dir].cachehint = IPSEC_PCBHINT_UNKNOWN;
       if (ipsec_setspidx(m, &pcbsp->sp_cache[dir].cacheidx, dir, 1) != 0) {
               return EINVAL;
       }
       pcbsp->sp_cache[dir].cachesp = sp;
       if (pcbsp->sp_cache[dir].cachesp) {
               /*
                * If the PCB is connected, we can remember a hint to
                * possibly short-circuit IPsec processing in other places.
                */
               if (pcbsp->sp_cacheflags & IPSEC_PCBSP_CONNECTED) {
                       switch (pcbsp->sp_cache[dir].cachesp->policy) {
                       case IPSEC_POLICY_NONE:
                       case IPSEC_POLICY_BYPASS:
                               pcbsp->sp_cache[dir].cachehint =
                                   IPSEC_PCBHINT_NO;
                               break;
                       default:
                               pcbsp->sp_cache[dir].cachehint =
                                   IPSEC_PCBHINT_YES;
                       }
               }
       }
       pcbsp->sp_cache[dir].cachegen = ipsec_spdgen;

       return 0;
}

static int
ipsec_invalpcbcache(struct inpcbpolicy *pcbsp, int dir)
{
       int i;

       KASSERT(inp_locked(pcbsp->sp_inp));

       for (i = IPSEC_DIR_INBOUND; i <= IPSEC_DIR_OUTBOUND; i++) {
               if (dir != IPSEC_DIR_ANY && i != dir)
                       continue;
               pcbsp->sp_cache[i].cachesp = NULL;
               pcbsp->sp_cache[i].cachehint = IPSEC_PCBHINT_UNKNOWN;
               pcbsp->sp_cache[i].cachegen = 0;
               memset(&pcbsp->sp_cache[i].cacheidx, 0,
                   sizeof(pcbsp->sp_cache[i].cacheidx));
       }
       return 0;
}

void
ipsec_pcbconn(struct inpcbpolicy *pcbsp)
{

       KASSERT(inp_locked(pcbsp->sp_inp));

       pcbsp->sp_cacheflags |= IPSEC_PCBSP_CONNECTED;
       ipsec_invalpcbcache(pcbsp, IPSEC_DIR_ANY);
}

void
ipsec_pcbdisconn(struct inpcbpolicy *pcbsp)
{

       KASSERT(inp_locked(pcbsp->sp_inp));

       pcbsp->sp_cacheflags &= ~IPSEC_PCBSP_CONNECTED;
       ipsec_invalpcbcache(pcbsp, IPSEC_DIR_ANY);
}

void
ipsec_invalpcbcacheall(void)
{

       if (ipsec_spdgen == UINT_MAX)
               ipsec_spdgen = 1;
       else
               ipsec_spdgen++;
}

/*
* Return a held reference to the default SP.
*/
static struct secpolicy *
key_get_default_sp(int af, const char *where, int tag)
{
       struct secpolicy *sp;

       KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_STAMP, "DP from %s:%u\n", where, tag);

       switch(af) {
       case AF_INET:
               sp = &ip4_def_policy;
               break;
#ifdef INET6
       case AF_INET6:
               sp = &ip6_def_policy;
               break;
#endif
       default:
               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_STAMP,
                   "unexpected protocol family %u\n", af);
               return NULL;
       }

       if (sp->policy != IPSEC_POLICY_DISCARD &&
           sp->policy != IPSEC_POLICY_NONE) {
               IPSECLOG(LOG_INFO, "fixed system default policy: %d->%d\n",
                   sp->policy, IPSEC_POLICY_NONE);
               sp->policy = IPSEC_POLICY_NONE;
       }
       KEY_SP_REF(sp);

       KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_STAMP, "DP returns SP:%p (%u)\n",
           sp, key_sp_refcnt(sp));
       return sp;
}

#define KEY_GET_DEFAULT_SP(af) \
       key_get_default_sp((af), __func__, __LINE__)

/*
* For OUTBOUND packet having a socket. Searching SPD for packet,
* and return a pointer to SP.
* OUT: NULL:   no appropriate SP found, the following value is set to error.
*              0       : bypass
*              EACCES  : discard packet.
*              ENOENT  : ipsec_acquire() in progress, maybe.
*              others  : error occurred.
*      others: a pointer to SP
*
* NOTE: IPv6 mapped address concern is implemented here.
*/
static struct secpolicy *
ipsec_getpolicybysock(struct mbuf *m, u_int dir, struct inpcb *inp,
   int *error)
{
       struct inpcbpolicy *pcbsp = NULL;
       struct secpolicy *currsp = NULL;        /* policy on socket */
       struct secpolicy *sp;
       int af;

       KASSERT(m != NULL);
       KASSERT(inp != NULL);
       KASSERT(error != NULL);
       KASSERTMSG(IPSEC_DIR_IS_INOROUT(dir), "invalid direction %u", dir);

       KASSERT(inp->inp_socket != NULL);
       KASSERT(inp_locked(inp));

       /* XXX FIXME inpcb vs socket*/
       af = inp->inp_af;
       KASSERTMSG(af == AF_INET || af == AF_INET6,
           "unexpected protocol family %u", af);

       KASSERT(inp->inp_sp != NULL);
       /* If we have a cached entry, and if it is still valid, use it. */
       IPSEC_STATINC(IPSEC_STAT_SPDCACHELOOKUP);
       currsp = ipsec_checkpcbcache(m, inp->inp_sp, dir);
       if (currsp) {
               *error = 0;
               return currsp;
       }
       IPSEC_STATINC(IPSEC_STAT_SPDCACHEMISS);

       switch (af) {
       case AF_INET:
#if defined(INET6)
       case AF_INET6:
#endif
               *error = ipsec_setspidx_inpcb(m, inp);
               pcbsp = inp->inp_sp;
               break;
       default:
               *error = EPFNOSUPPORT;
               break;
       }
       if (*error)
               return NULL;

       KASSERT(pcbsp != NULL);
       switch (dir) {
       case IPSEC_DIR_INBOUND:
               currsp = pcbsp->sp_in;
               break;
       case IPSEC_DIR_OUTBOUND:
               currsp = pcbsp->sp_out;
               break;
       }
       KASSERT(currsp != NULL);

       if (pcbsp->priv) {      /* when privileged socket */
               switch (currsp->policy) {
               case IPSEC_POLICY_BYPASS:
               case IPSEC_POLICY_IPSEC:
                       KEY_SP_REF(currsp);
                       sp = currsp;
                       break;

               case IPSEC_POLICY_ENTRUST:
                       /* look for a policy in SPD */
                       if (key_havesp(dir))
                               sp = KEY_LOOKUP_SP_BYSPIDX(&currsp->spidx, dir);
                       else
                               sp = NULL;
                       if (sp == NULL)         /* no SP found */
                               sp = KEY_GET_DEFAULT_SP(af);
                       break;

               default:
                       IPSECLOG(LOG_ERR, "Invalid policy for PCB %d\n",
                           currsp->policy);
                       *error = EINVAL;
                       return NULL;
               }
       } else {                                /* unpriv, SPD has policy */
               if (key_havesp(dir))
                       sp = KEY_LOOKUP_SP_BYSPIDX(&currsp->spidx, dir);
               else
                       sp = NULL;
               if (sp == NULL) {               /* no SP found */
                       switch (currsp->policy) {
                       case IPSEC_POLICY_BYPASS:
                               IPSECLOG(LOG_ERR, "Illegal policy for "
                                   "non-privileged defined %d\n",
                                   currsp->policy);
                               *error = EINVAL;
                               return NULL;

                       case IPSEC_POLICY_ENTRUST:
                               sp = KEY_GET_DEFAULT_SP(af);
                               break;

                       case IPSEC_POLICY_IPSEC:
                               KEY_SP_REF(currsp);
                               sp = currsp;
                               break;

                       default:
                               IPSECLOG(LOG_ERR, "Invalid policy for "
                                   "PCB %d\n", currsp->policy);
                               *error = EINVAL;
                               return NULL;
                       }
               }
       }
       KASSERTMSG(sp != NULL, "null SP (priv %u policy %u", pcbsp->priv,
           currsp->policy);
       KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_STAMP,
           "DP (priv %u policy %u) allocates SP:%p (refcnt %u)\n",
           pcbsp->priv, currsp->policy, sp, key_sp_refcnt(sp));
       ipsec_fillpcbcache(pcbsp, m, sp, dir);
       return sp;
}

/*
* For FORWARDING packet or OUTBOUND without a socket. Searching SPD for packet,
* and return a pointer to SP.
* OUT: positive: a pointer to the entry for security policy leaf matched.
*      NULL:   no appropriate SP found, the following value is set to error.
*              0       : bypass
*              EACCES  : discard packet.
*              ENOENT  : ipsec_acquire() in progress, maybe.
*              others  : error occurred.
*/
static struct secpolicy *
ipsec_getpolicybyaddr(struct mbuf *m, u_int dir, int flag, int *error)
{
       struct secpolicyindex spidx;
       struct secpolicy *sp;

       KASSERT(m != NULL);
       KASSERT(error != NULL);
       KASSERTMSG(IPSEC_DIR_IS_INOROUT(dir), "invalid direction %u", dir);

       sp = NULL;

       /* Make an index to look for a policy. */
       *error = ipsec_setspidx(m, &spidx, dir, 1);
       if (*error != 0) {
               IPSECLOG(LOG_DEBUG, "setpidx failed, dir %u flag %u\n", dir, flag);
               memset(&spidx, 0, sizeof(spidx));
               return NULL;
       }

       spidx.dir = dir;

       if (key_havesp(dir)) {
               sp = KEY_LOOKUP_SP_BYSPIDX(&spidx, dir);
       }
       if (sp == NULL) {
               /* no SP found, use system default */
               sp = KEY_GET_DEFAULT_SP(spidx.dst.sa.sa_family);
       }

       KASSERT(sp != NULL);
       return sp;
}

static struct secpolicy *
ipsec_checkpolicy(struct mbuf *m, u_int dir, u_int flag, int *error,
   struct inpcb *inp)
{
       struct secpolicy *sp;

       *error = 0;

       if (inp == NULL) {
               sp = ipsec_getpolicybyaddr(m, dir, flag, error);
       } else {
               KASSERT(inp->inp_socket != NULL);
               sp = ipsec_getpolicybysock(m, dir, inp, error);
       }
       if (sp == NULL) {
               KASSERTMSG(*error != 0, "getpolicy failed w/o error");
               IPSEC_STATINC(IPSEC_STAT_OUT_INVAL);
               return NULL;
       }
       KASSERTMSG(*error == 0, "sp w/ error set to %u", *error);

       switch (sp->policy) {
       case IPSEC_POLICY_ENTRUST:
       default:
               printf("%s: invalid policy %u\n", __func__, sp->policy);
               /* fall thru... */
       case IPSEC_POLICY_DISCARD:
               IPSEC_STATINC(IPSEC_STAT_OUT_POLVIO);
               *error = -EINVAL;       /* packet is discarded by caller */
               break;
       case IPSEC_POLICY_BYPASS:
       case IPSEC_POLICY_NONE:
               KEY_SP_UNREF(&sp);
               sp = NULL;              /* NB: force NULL result */
               break;
       case IPSEC_POLICY_IPSEC:
               KASSERT(sp->req != NULL);
               break;
       }

       if (*error != 0) {
               KEY_SP_UNREF(&sp);
               sp = NULL;
               IPSECLOG(LOG_DEBUG, "done, error %d\n", *error);
       }

       return sp;
}

int
ipsec4_output(struct mbuf *m, struct inpcb *inp, int flags,
   u_long *mtu, bool *natt_frag, bool *done, bool *count_drop)
{
       struct secpolicy *sp = NULL;
       u_long _mtu = 0;
       int error;

       /*
        * Check the security policy (SP) for the packet and, if required,
        * do IPsec-related processing.  There are two cases here; the first
        * time a packet is sent through it will be untagged and handled by
        * ipsec_checkpolicy().  If the packet is resubmitted to ip_output
        * (e.g. after AH, ESP, etc. processing), there will be a tag to
        * bypass the lookup and related policy checking.
        */
       if (ipsec_outdone(m)) {
               return 0;
       }
       if (inp && ipsec_pcb_skip_ipsec(inp->inp_sp, IPSEC_DIR_OUTBOUND)) {
               return 0;
       }
       sp = ipsec_checkpolicy(m, IPSEC_DIR_OUTBOUND, flags, &error, inp);

       /*
        * There are four return cases:
        *      sp != NULL                    apply IPsec policy
        *      sp == NULL, error == 0        no IPsec handling needed
        *      sp == NULL, error == -EINVAL  discard packet w/o error
        *      sp == NULL, error != 0        discard packet, report error
        */
       if (sp == NULL) {
               if (error) {
                       /*
                        * Hack: -EINVAL is used to signal that a packet
                        * should be silently discarded.  This is typically
                        * because we asked key management for an SA and
                        * it was delayed (e.g. kicked up to IKE).
                        */
                       if (error == -EINVAL)
                               error = 0;
                       m_freem(m);
                       *done = true;
                       *count_drop = true;
                       return error;
               }
               /* No IPsec processing for this packet. */
               return 0;
       }

       /*
        * Do delayed checksums now because we send before
        * this is done in the normal processing path.
        */
       if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4)) {
               in_undefer_cksum_tcpudp(m);
               m->m_pkthdr.csum_flags &= ~(M_CSUM_TCPv4|M_CSUM_UDPv4);
       }

       error = ipsec4_process_packet(m, sp->req, &_mtu);
       if (error == 0 && _mtu != 0) {
               /*
                * NAT-T ESP fragmentation: do not do IPSec processing
                * now, we will do it on each fragmented packet.
                */
               *mtu = _mtu;
               *natt_frag = true;
               KEY_SP_UNREF(&sp);
               return 0;
       }

       /*
        * Preserve KAME behaviour: ENOENT can be returned
        * when an SA acquire is in progress.  Don't propagate
        * this to user-level; it confuses applications.
        *
        * XXX this will go away when the SADB is redone.
        */
       if (error == ENOENT)
               error = 0;
       KEY_SP_UNREF(&sp);
       *done = true;
       return error;
}

int
ipsec_ip_input_checkpolicy(struct mbuf *m, bool forward)
{
       struct secpolicy *sp;
       int error;

       error = ipsec_in_reject(m, NULL);
       if (error) {
               return EINVAL;
       }

       if (!forward || !(m->m_flags & M_CANFASTFWD)) {
               return 0;
       }

       /*
        * Peek at the outbound SP for this packet to determine if
        * it is a Fast Forward candidate.
        */
       sp = ipsec_checkpolicy(m, IPSEC_DIR_OUTBOUND, IP_FORWARDING,
           &error, NULL);
       if (sp != NULL) {
               m->m_flags &= ~M_CANFASTFWD;
               KEY_SP_UNREF(&sp);
       }

       return 0;
}

/*
* If the packet is routed over IPsec tunnel, tell the originator the
* tunnel MTU.
*     tunnel MTU = if MTU - sizeof(IP) - ESP/AH hdrsiz
*
* XXX: Quick hack!!!
*
* XXX: And what if the MTU goes negative?
*/
void
ipsec_mtu(struct mbuf *m, int *destmtu)
{
       struct secpolicy *sp;
       size_t ipsechdr;
       int error;

       sp = ipsec_getpolicybyaddr(m, IPSEC_DIR_OUTBOUND, IP_FORWARDING,
           &error);
       if (sp == NULL) {
               return;
       }

       /* Count IPsec header size. */
       ipsechdr = ipsec_sp_hdrsiz(sp, m);

       /*
        * Find the correct route for outer IP header, compute tunnel MTU.
        */
       if (sp->req) {
               struct secasvar *sav;

               sav = ipsec_lookup_sa(sp->req, m);
               if (sav != NULL) {
                       struct route *ro;
                       struct rtentry *rt;

                       ro = &sav->sah->sa_route;
                       rt = rtcache_validate(ro);
                       if (rt && rt->rt_ifp) {
                               *destmtu = rt->rt_rmx.rmx_mtu ?
                                   rt->rt_rmx.rmx_mtu : rt->rt_ifp->if_mtu;
                               *destmtu -= ipsechdr;
                       }
                       rtcache_unref(rt, ro);
                       KEY_SA_UNREF(&sav);
               }
       }
       KEY_SP_UNREF(&sp);
}

static int
ipsec_setspidx_inpcb(struct mbuf *m, struct inpcb *inp)
{
       int error;

       KASSERT(inp != NULL);
       KASSERT(inp->inp_sp != NULL);
       KASSERT(inp->inp_sp->sp_out != NULL);
       KASSERT(inp->inp_sp->sp_in != NULL);

       error = ipsec_setspidx(m, &inp->inp_sp->sp_in->spidx,
           IPSEC_DIR_INBOUND, 1);
       if (error == 0) {
               inp->inp_sp->sp_out->spidx = inp->inp_sp->sp_in->spidx;
               inp->inp_sp->sp_out->spidx.dir = IPSEC_DIR_OUTBOUND;
       } else {
               memset(&inp->inp_sp->sp_in->spidx, 0,
                   sizeof(inp->inp_sp->sp_in->spidx));
               memset(&inp->inp_sp->sp_out->spidx, 0,
                   sizeof(inp->inp_sp->sp_out->spidx));
       }
       return error;
}

/*
* configure security policy index (src/dst/proto/sport/dport)
* by looking at the content of mbuf.
* the caller is responsible for error recovery (like clearing up spidx).
*/
static int
ipsec_setspidx(struct mbuf *m, struct secpolicyindex *spidx, int dir,
   int needport)
{
       struct ip *ip = NULL;
       struct ip ipbuf;
       u_int v;
       int error;

       KASSERT(m != NULL);
       M_VERIFY_PACKET(m);

       if (m->m_pkthdr.len < sizeof(struct ip)) {
               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DUMP,
                   "pkthdr.len(%d) < sizeof(struct ip), ignored.\n",
                   m->m_pkthdr.len);
               return EINVAL;
       }

       memset(spidx, 0, sizeof(*spidx));
       spidx->dir = dir;

       if (m->m_len >= sizeof(*ip)) {
               ip = mtod(m, struct ip *);
       } else {
               m_copydata(m, 0, sizeof(ipbuf), &ipbuf);
               ip = &ipbuf;
       }
       v = ip->ip_v;
       switch (v) {
       case 4:
               error = ipsec4_setspidx_ipaddr(m, spidx);
               if (error)
                       return error;
               ipsec4_get_ulp(m, spidx, needport);
               return 0;
#ifdef INET6
       case 6:
               if (m->m_pkthdr.len < sizeof(struct ip6_hdr)) {
                       KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DUMP,
                           "pkthdr.len(%d) < sizeof(struct ip6_hdr), "
                           "ignored.\n", m->m_pkthdr.len);
                       return EINVAL;
               }
               error = ipsec6_setspidx_ipaddr(m, spidx);
               if (error)
                       return error;
               ipsec6_get_ulp(m, spidx, needport);
               return 0;
#endif
       default:
               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DUMP,
                   "unknown IP version %u, ignored.\n", v);
               return EINVAL;
       }
}

static void
ipsec4_get_ulp(struct mbuf *m, struct secpolicyindex *spidx, int needport)
{
       u_int8_t nxt;
       int off;

       KASSERT(m != NULL);
       KASSERTMSG(m->m_pkthdr.len >= sizeof(struct ip), "packet too short");

       /* NB: ip_input() flips it into host endian XXX need more checking */
       if (m->m_len >= sizeof(struct ip)) {
               struct ip *ip = mtod(m, struct ip *);
               if (ip->ip_off & htons(IP_MF | IP_OFFMASK))
                       goto done;
               off = ip->ip_hl << 2;
               nxt = ip->ip_p;
       } else {
               struct ip ih;

               m_copydata(m, 0, sizeof(struct ip), &ih);
               if (ih.ip_off & htons(IP_MF | IP_OFFMASK))
                       goto done;
               off = ih.ip_hl << 2;
               nxt = ih.ip_p;
       }

       while (off < m->m_pkthdr.len) {
               struct ip6_ext ip6e;
               struct tcphdr th;
               struct udphdr uh;
               struct icmp icmph;

               switch (nxt) {
               case IPPROTO_TCP:
                       spidx->ul_proto = nxt;
                       if (!needport)
                               goto done_proto;
                       if (off + sizeof(struct tcphdr) > m->m_pkthdr.len)
                               goto done;
                       m_copydata(m, off, sizeof(th), &th);
                       spidx->src.sin.sin_port = th.th_sport;
                       spidx->dst.sin.sin_port = th.th_dport;
                       return;
               case IPPROTO_UDP:
                       spidx->ul_proto = nxt;
                       if (!needport)
                               goto done_proto;
                       if (off + sizeof(struct udphdr) > m->m_pkthdr.len)
                               goto done;
                       m_copydata(m, off, sizeof(uh), &uh);
                       spidx->src.sin.sin_port = uh.uh_sport;
                       spidx->dst.sin.sin_port = uh.uh_dport;
                       return;
               case IPPROTO_AH:
                       if (off + sizeof(ip6e) > m->m_pkthdr.len)
                               goto done;
                       /* XXX sigh, this works but is totally bogus */
                       m_copydata(m, off, sizeof(ip6e), &ip6e);
                       off += (ip6e.ip6e_len + 2) << 2;
                       nxt = ip6e.ip6e_nxt;
                       break;
               case IPPROTO_ICMP:
                       spidx->ul_proto = nxt;
                       if (off + sizeof(struct icmp) > m->m_pkthdr.len)
                               goto done;
                       m_copydata(m, off, sizeof(icmph), &icmph);
                       ((struct sockaddr_in *)&spidx->src)->sin_port =
                           htons((uint16_t)icmph.icmp_type);
                       ((struct sockaddr_in *)&spidx->dst)->sin_port =
                           htons((uint16_t)icmph.icmp_code);
                       return;
               default:
                       /* XXX intermediate headers??? */
                       spidx->ul_proto = nxt;
                       goto done_proto;
               }
       }
done:
       spidx->ul_proto = IPSEC_ULPROTO_ANY;
done_proto:
       spidx->src.sin.sin_port = IPSEC_PORT_ANY;
       spidx->dst.sin.sin_port = IPSEC_PORT_ANY;
}

static int
ipsec4_setspidx_ipaddr(struct mbuf *m, struct secpolicyindex *spidx)
{
       static const struct sockaddr_in template = {
               sizeof(struct sockaddr_in),
               AF_INET,
               0, { 0 }, { 0, 0, 0, 0, 0, 0, 0, 0 }
       };

       spidx->src.sin = template;
       spidx->dst.sin = template;

       if (m->m_len < sizeof(struct ip)) {
               m_copydata(m, offsetof(struct ip, ip_src),
                   sizeof(struct in_addr), &spidx->src.sin.sin_addr);
               m_copydata(m, offsetof(struct ip, ip_dst),
                   sizeof(struct in_addr), &spidx->dst.sin.sin_addr);
       } else {
               struct ip *ip = mtod(m, struct ip *);
               spidx->src.sin.sin_addr = ip->ip_src;
               spidx->dst.sin.sin_addr = ip->ip_dst;
       }

       spidx->prefs = sizeof(struct in_addr) << 3;
       spidx->prefd = sizeof(struct in_addr) << 3;

       return 0;
}

#ifdef INET6
static void
ipsec6_get_ulp(struct mbuf *m, struct secpolicyindex *spidx, int needport)
{
       int off, nxt;
       struct tcphdr th;
       struct udphdr uh;
       struct icmp6_hdr icmph;

       KASSERT(m != NULL);

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DUMP)) {
               kdebug_mbuf(__func__, m);
       }

       /* set default */
       spidx->ul_proto = IPSEC_ULPROTO_ANY;
       ((struct sockaddr_in6 *)&spidx->src)->sin6_port = IPSEC_PORT_ANY;
       ((struct sockaddr_in6 *)&spidx->dst)->sin6_port = IPSEC_PORT_ANY;

       nxt = -1;
       off = ip6_lasthdr(m, 0, IPPROTO_IPV6, &nxt);
       if (off < 0 || m->m_pkthdr.len < off)
               return;

       switch (nxt) {
       case IPPROTO_TCP:
               spidx->ul_proto = nxt;
               if (!needport)
                       break;
               if (off + sizeof(struct tcphdr) > m->m_pkthdr.len)
                       break;
               m_copydata(m, off, sizeof(th), &th);
               ((struct sockaddr_in6 *)&spidx->src)->sin6_port = th.th_sport;
               ((struct sockaddr_in6 *)&spidx->dst)->sin6_port = th.th_dport;
               break;
       case IPPROTO_UDP:
               spidx->ul_proto = nxt;
               if (!needport)
                       break;
               if (off + sizeof(struct udphdr) > m->m_pkthdr.len)
                       break;
               m_copydata(m, off, sizeof(uh), &uh);
               ((struct sockaddr_in6 *)&spidx->src)->sin6_port = uh.uh_sport;
               ((struct sockaddr_in6 *)&spidx->dst)->sin6_port = uh.uh_dport;
               break;
       case IPPROTO_ICMPV6:
               spidx->ul_proto = nxt;
               if (off + sizeof(struct icmp6_hdr) > m->m_pkthdr.len)
                       break;
               m_copydata(m, off, sizeof(icmph), &icmph);
               ((struct sockaddr_in6 *)&spidx->src)->sin6_port =
                   htons((uint16_t)icmph.icmp6_type);
               ((struct sockaddr_in6 *)&spidx->dst)->sin6_port =
                   htons((uint16_t)icmph.icmp6_code);
               break;
       default:
               /* XXX intermediate headers??? */
               spidx->ul_proto = nxt;
               break;
       }
}

static int
ipsec6_setspidx_ipaddr(struct mbuf *m, struct secpolicyindex *spidx)
{
       struct ip6_hdr *ip6 = NULL;
       struct ip6_hdr ip6buf;
       struct sockaddr_in6 *sin6;

       if (m->m_len >= sizeof(*ip6)) {
               ip6 = mtod(m, struct ip6_hdr *);
       } else {
               m_copydata(m, 0, sizeof(ip6buf), &ip6buf);
               ip6 = &ip6buf;
       }

       sin6 = (struct sockaddr_in6 *)&spidx->src;
       memset(sin6, 0, sizeof(*sin6));
       sin6->sin6_family = AF_INET6;
       sin6->sin6_len = sizeof(struct sockaddr_in6);
       memcpy(&sin6->sin6_addr, &ip6->ip6_src, sizeof(ip6->ip6_src));
       if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src)) {
               sin6->sin6_addr.s6_addr16[1] = 0;
               sin6->sin6_scope_id = ntohs(ip6->ip6_src.s6_addr16[1]);
       }
       spidx->prefs = sizeof(struct in6_addr) << 3;

       sin6 = (struct sockaddr_in6 *)&spidx->dst;
       memset(sin6, 0, sizeof(*sin6));
       sin6->sin6_family = AF_INET6;
       sin6->sin6_len = sizeof(struct sockaddr_in6);
       memcpy(&sin6->sin6_addr, &ip6->ip6_dst, sizeof(ip6->ip6_dst));
       if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst)) {
               sin6->sin6_addr.s6_addr16[1] = 0;
               sin6->sin6_scope_id = ntohs(ip6->ip6_dst.s6_addr16[1]);
       }
       spidx->prefd = sizeof(struct in6_addr) << 3;

       return 0;
}
#endif

static void
ipsec_delpcbpolicy(struct inpcbpolicy *p)
{

       kmem_intr_free(p, sizeof(*p));
}

int
ipsec_init_pcbpolicy(struct socket *so, struct inpcbpolicy **policy)
{
       struct inpcbpolicy *new;

       KASSERT(so != NULL);
       KASSERT(policy != NULL);

       new = kmem_intr_zalloc(sizeof(*new), KM_NOSLEEP);
       if (new == NULL) {
               IPSECLOG(LOG_DEBUG, "No more memory.\n");
               return ENOBUFS;
       }

       if (IPSEC_PRIVILEGED_SO(so))
               new->priv = 1;
       else
               new->priv = 0;

       /*
        * Set dummy SPs. Actual SPs will be allocated later if needed.
        */
       new->sp_in = &ipsec_dummy_sp;
       new->sp_out = &ipsec_dummy_sp;

       *policy = new;

       return 0;
}

static void
ipsec_destroy_policy(struct secpolicy *sp)
{

       if (sp == &ipsec_dummy_sp) {
               ; /* It's dummy. No need to free it. */
       } else {
               /*
                * We cannot destroy here because it can be called in
                * softint. So mark the SP as DEAD and let the timer
                * destroy it. See key_timehandler_spd.
                */
               sp->state = IPSEC_SPSTATE_DEAD;
       }
}

int
ipsec_set_policy(struct inpcb *inp, const void *request, size_t len,
   kauth_cred_t cred)
{
       const struct sadb_x_policy *xpl;
       struct secpolicy *newsp, *oldsp;
       struct secpolicy **policy;
       int error;

       KASSERT(!cpu_softintr_p());
       KASSERT(inp != NULL);
       KASSERT(inp_locked(inp));
       KASSERT(request != NULL);

       if (len < sizeof(*xpl))
               return EINVAL;
       xpl = (const struct sadb_x_policy *)request;

       KASSERT(inp->inp_sp != NULL);

       /* select direction */
       switch (xpl->sadb_x_policy_dir) {
       case IPSEC_DIR_INBOUND:
               policy = &inp->inp_sp->sp_in;
               break;
       case IPSEC_DIR_OUTBOUND:
               policy = &inp->inp_sp->sp_out;
               break;
       default:
               IPSECLOG(LOG_ERR, "invalid direction=%u\n",
                   xpl->sadb_x_policy_dir);
               return EINVAL;
       }

       /* sanity check. */
       if (policy == NULL || *policy == NULL)
               return EINVAL;

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DUMP)) {
               kdebug_sadb_xpolicy("set passed policy", request);
       }

       /* check policy type */
       /* ipsec_set_policy() accepts IPSEC, ENTRUST and BYPASS. */
       if (xpl->sadb_x_policy_type == IPSEC_POLICY_DISCARD ||
           xpl->sadb_x_policy_type == IPSEC_POLICY_NONE)
               return EINVAL;

       /* check privileged socket */
       if (xpl->sadb_x_policy_type == IPSEC_POLICY_BYPASS) {
               error = kauth_authorize_network(cred, KAUTH_NETWORK_IPSEC,
                   KAUTH_REQ_NETWORK_IPSEC_BYPASS, NULL, NULL, NULL);
               if (error)
                       return error;
       }

       /* allocation new SP entry */
       if ((newsp = key_msg2sp(xpl, len, &error)) == NULL)
               return error;

       key_init_sp(newsp);
       newsp->created = time_uptime;
       /* Insert the global list for SPs for sockets */
       key_socksplist_add(newsp);

       /* clear old SP and set new SP */
       oldsp = *policy;
       *policy = newsp;
       ipsec_destroy_policy(oldsp);

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DUMP)) {
               printf("%s: new policy\n", __func__);
               kdebug_secpolicy(newsp);
       }

       return 0;
}

int
ipsec_get_policy(struct inpcb *inp, const void *request, size_t len,
   struct mbuf **mp)
{
       const struct sadb_x_policy *xpl;
       struct secpolicy *policy;

       /* sanity check. */
       if (inp == NULL || request == NULL || mp == NULL)
               return EINVAL;
       KASSERT(inp->inp_sp != NULL);
       if (len < sizeof(*xpl))
               return EINVAL;
       xpl = (const struct sadb_x_policy *)request;

       /* select direction */
       switch (xpl->sadb_x_policy_dir) {
       case IPSEC_DIR_INBOUND:
               policy = inp->inp_sp->sp_in;
               break;
       case IPSEC_DIR_OUTBOUND:
               policy = inp->inp_sp->sp_out;
               break;
       default:
               IPSECLOG(LOG_ERR, "invalid direction=%u\n",
                   xpl->sadb_x_policy_dir);
               return EINVAL;
       }

       if (policy == NULL)
               return EINVAL;

       *mp = key_sp2msg(policy, M_NOWAIT);
       if (!*mp) {
               IPSECLOG(LOG_DEBUG, "No more memory.\n");
               return ENOBUFS;
       }

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DUMP)) {
               kdebug_mbuf(__func__, *mp);
       }

       return 0;
}

int
ipsec_delete_pcbpolicy(struct inpcb *inp)
{

       KASSERT(inp != NULL);

       if (inp->inp_sp == NULL)
               return 0;

       if (inp->inp_sp->sp_in != NULL)
               ipsec_destroy_policy(inp->inp_sp->sp_in);

       if (inp->inp_sp->sp_out != NULL)
               ipsec_destroy_policy(inp->inp_sp->sp_out);

       ipsec_invalpcbcache(inp->inp_sp, IPSEC_DIR_ANY);

       ipsec_delpcbpolicy(inp->inp_sp);
       inp->inp_sp = NULL;

       return 0;
}

/*
* Return the current level (either IPSEC_LEVEL_USE or IPSEC_LEVEL_REQUIRE).
*/
u_int
ipsec_get_reqlevel(const struct ipsecrequest *isr)
{
       u_int level = 0;
       u_int esp_trans_deflev, esp_net_deflev;
       u_int ah_trans_deflev, ah_net_deflev;

       KASSERT(isr != NULL);
       KASSERT(isr->sp != NULL);
       KASSERTMSG(
           isr->sp->spidx.src.sa.sa_family == isr->sp->spidx.dst.sa.sa_family,
           "af family mismatch, src %u, dst %u",
           isr->sp->spidx.src.sa.sa_family, isr->sp->spidx.dst.sa.sa_family);

/* XXX note that we have ipseclog() expanded here - code sync issue */
#define IPSEC_CHECK_DEFAULT(lev)                                        \
   (((lev) != IPSEC_LEVEL_USE && (lev) != IPSEC_LEVEL_REQUIRE          \
   && (lev) != IPSEC_LEVEL_UNIQUE) ?                                   \
       (ipsec_debug ? log(LOG_INFO, "fixed system default level " #lev \
       ":%d->%d\n", (lev), IPSEC_LEVEL_REQUIRE) : (void)0),            \
       (lev) = IPSEC_LEVEL_REQUIRE, (lev)                              \
   : (lev))

       /* set default level */
       switch (((struct sockaddr *)&isr->sp->spidx.src)->sa_family) {
#ifdef INET
       case AF_INET:
               esp_trans_deflev = IPSEC_CHECK_DEFAULT(ip4_esp_trans_deflev);
               esp_net_deflev = IPSEC_CHECK_DEFAULT(ip4_esp_net_deflev);
               ah_trans_deflev = IPSEC_CHECK_DEFAULT(ip4_ah_trans_deflev);
               ah_net_deflev = IPSEC_CHECK_DEFAULT(ip4_ah_net_deflev);
               break;
#endif
#ifdef INET6
       case AF_INET6:
               esp_trans_deflev = IPSEC_CHECK_DEFAULT(ip6_esp_trans_deflev);
               esp_net_deflev = IPSEC_CHECK_DEFAULT(ip6_esp_net_deflev);
               ah_trans_deflev = IPSEC_CHECK_DEFAULT(ip6_ah_trans_deflev);
               ah_net_deflev = IPSEC_CHECK_DEFAULT(ip6_ah_net_deflev);
               break;
#endif
       default:
               panic("%s: unknown af %u", __func__,
                   isr->sp->spidx.src.sa.sa_family);
       }

#undef IPSEC_CHECK_DEFAULT

       /* set level */
       switch (isr->level) {
       case IPSEC_LEVEL_DEFAULT:
               switch (isr->saidx.proto) {
               case IPPROTO_ESP:
                       if (isr->saidx.mode == IPSEC_MODE_TUNNEL)
                               level = esp_net_deflev;
                       else
                               level = esp_trans_deflev;
                       break;
               case IPPROTO_AH:
                       if (isr->saidx.mode == IPSEC_MODE_TUNNEL)
                               level = ah_net_deflev;
                       else
                               level = ah_trans_deflev;
                       break;
               case IPPROTO_IPCOMP:
                       /*
                        * we don't really care, as IPcomp document says that
                        * we shouldn't compress small packets
                        */
                       level = IPSEC_LEVEL_USE;
                       break;
               default:
                       panic("%s: Illegal protocol defined %u", __func__,
                           isr->saidx.proto);
               }
               break;

       case IPSEC_LEVEL_USE:
       case IPSEC_LEVEL_REQUIRE:
               level = isr->level;
               break;
       case IPSEC_LEVEL_UNIQUE:
               level = IPSEC_LEVEL_REQUIRE;
               break;

       default:
               panic("%s: Illegal IPsec level %u", __func__, isr->level);
       }

       return level;
}

/*
* Check security policy requirements against the actual packet contents.
*
* If the SP requires an IPsec packet, and the packet was neither AH nor ESP,
* then kick it.
*/
static int
ipsec_sp_reject(const struct secpolicy *sp, const struct mbuf *m)
{
       struct ipsecrequest *isr;

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DATA)) {
               printf("%s: using SP\n", __func__);
               kdebug_secpolicy(sp);
       }

       /* check policy */
       switch (sp->policy) {
       case IPSEC_POLICY_DISCARD:
               return 1;
       case IPSEC_POLICY_BYPASS:
       case IPSEC_POLICY_NONE:
               return 0;
       }

       KASSERTMSG(sp->policy == IPSEC_POLICY_IPSEC,
           "invalid policy %u", sp->policy);

       /* XXX should compare policy against ipsec header history */

       for (isr = sp->req; isr != NULL; isr = isr->next) {
               if (ipsec_get_reqlevel(isr) != IPSEC_LEVEL_REQUIRE)
                       continue;
               switch (isr->saidx.proto) {
               case IPPROTO_ESP:
                       if ((m->m_flags & M_DECRYPTED) == 0) {
                               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DUMP,
                                   "ESP m_flags:%x\n", m->m_flags);
                               return 1;
                       }
                       break;
               case IPPROTO_AH:
                       if ((m->m_flags & M_AUTHIPHDR) == 0) {
                               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DUMP,
                                   "AH m_flags:%x\n", m->m_flags);
                               return 1;
                       }
                       break;
               case IPPROTO_IPCOMP:
                       /*
                        * We don't really care, as IPcomp document
                        * says that we shouldn't compress small
                        * packets, IPComp policy should always be
                        * treated as being in "use" level.
                        */
                       break;
               }
       }

       return 0;
}

/*
* Check security policy requirements.
*/
int
ipsec_in_reject(struct mbuf *m, struct inpcb *inp)
{
       struct secpolicy *sp;
       int error;
       int result;

       KASSERT(m != NULL);

       if (inp == NULL)
               sp = ipsec_getpolicybyaddr(m, IPSEC_DIR_INBOUND,
                   IP_FORWARDING, &error);
       else
               sp = ipsec_getpolicybysock(m, IPSEC_DIR_INBOUND,
                   inp, &error);

       if (sp != NULL) {
               result = ipsec_sp_reject(sp, m);
               if (result)
                       IPSEC_STATINC(IPSEC_STAT_IN_POLVIO);
               KEY_SP_UNREF(&sp);
       } else {
               result = 0;
       }
       return result;
}

/*
* Compute the byte size to be occupied by the IPsec header. If it is
* tunneled, it includes the size of outer IP header.
*/
static size_t
ipsec_sp_hdrsiz(const struct secpolicy *sp, const struct mbuf *m)
{
       struct ipsecrequest *isr;
       size_t siz;

       if (KEYDEBUG_ON(KEYDEBUG_IPSEC_DATA)) {
               printf("%s: using SP\n", __func__);
               kdebug_secpolicy(sp);
       }

       switch (sp->policy) {
       case IPSEC_POLICY_DISCARD:
       case IPSEC_POLICY_BYPASS:
       case IPSEC_POLICY_NONE:
               return 0;
       }

       KASSERTMSG(sp->policy == IPSEC_POLICY_IPSEC,
           "invalid policy %u", sp->policy);

       siz = 0;
       for (isr = sp->req; isr != NULL; isr = isr->next) {
               size_t clen = 0;
               struct secasvar *sav;

               switch (isr->saidx.proto) {
               case IPPROTO_ESP:
                       sav = ipsec_lookup_sa(isr, m);
                       if (sav != NULL) {
                               clen = esp_hdrsiz(sav);
                               KEY_SA_UNREF(&sav);
                       } else
                               clen = esp_hdrsiz(NULL);
                       break;
               case IPPROTO_AH:
                       sav = ipsec_lookup_sa(isr, m);
                       if (sav != NULL) {
                               clen = ah_hdrsiz(sav);
                               KEY_SA_UNREF(&sav);
                       } else
                               clen = ah_hdrsiz(NULL);
                       break;
               case IPPROTO_IPCOMP:
                       clen = sizeof(struct ipcomp);
                       break;
               }

               if (isr->saidx.mode == IPSEC_MODE_TUNNEL) {
                       switch (isr->saidx.dst.sa.sa_family) {
                       case AF_INET:
                               clen += sizeof(struct ip);
                               break;
#ifdef INET6
                       case AF_INET6:
                               clen += sizeof(struct ip6_hdr);
                               break;
#endif
                       default:
                               IPSECLOG(LOG_ERR, "unknown AF %d in "
                                   "IPsec tunnel SA\n",
                                   ((const struct sockaddr *)&isr->saidx.dst)
                                   ->sa_family);
                               break;
                       }
               }
               siz += clen;
       }

       return siz;
}

size_t
ipsec_hdrsiz(struct mbuf *m, u_int dir, struct inpcb *inp)
{
       struct secpolicy *sp;
       int error;
       size_t size;

       KASSERT(m != NULL);
       KASSERTMSG(inp == NULL || inp->inp_socket != NULL,
           "socket w/o inpcb");

       if (inp == NULL)
               sp = ipsec_getpolicybyaddr(m, dir, IP_FORWARDING, &error);
       else
               sp = ipsec_getpolicybysock(m, dir, inp, &error);

       if (sp != NULL) {
               size = ipsec_sp_hdrsiz(sp, m);
               KEYDEBUG_PRINTF(KEYDEBUG_IPSEC_DATA, "size:%zu.\n", size);
               KEY_SP_UNREF(&sp);
       } else {
               size = 0;
       }

       return size;
}

/*
* Check the variable replay window.
* ipsec_chkreplay() performs replay check before ICV verification.
* ipsec_updatereplay() updates replay bitmap.  This must be called after
* ICV verification (it also performs replay check, which is usually done
* beforehand).
* 0 (zero) is returned if packet disallowed, 1 if packet permitted.
*
* based on RFC 2401.
*/
int
ipsec_chkreplay(u_int32_t seq, const struct secasvar *sav)
{
       const struct secreplay *replay;
       u_int32_t diff;
       int fr;
       u_int32_t wsizeb;       /* constant: bits of window size */
       int frlast;             /* constant: last frame */

       KASSERT(sav != NULL);
       KASSERT(sav->replay != NULL);

       replay = sav->replay;

       if (replay->wsize == 0)
               return 1;       /* no need to check replay. */

       /* constant */
       frlast = replay->wsize - 1;
       wsizeb = replay->wsize << 3;

       /* sequence number of 0 is invalid */
       if (seq == 0)
               return 0;

       /* first time is always okay */
       if (replay->count == 0)
               return 1;

       if (seq > replay->lastseq) {
               /* larger sequences are okay */
               return 1;
       } else {
               /* seq is equal or less than lastseq. */
               diff = replay->lastseq - seq;

               /* over range to check, i.e. too old or wrapped */
               if (diff >= wsizeb)
                       return 0;

               fr = frlast - diff / 8;

               /* this packet already seen ? */
               if ((replay->bitmap)[fr] & (1 << (diff % 8)))
                       return 0;

               /* out of order but good */
               return 1;
       }
}

/*
* check replay counter whether to update or not.
* OUT: 0:      OK
*      1:      NG
*/
int
ipsec_updatereplay(u_int32_t seq, const struct secasvar *sav)
{
       struct secreplay *replay;
       u_int32_t diff;
       int fr;
       u_int32_t wsizeb;       /* constant: bits of window size */
       int frlast;             /* constant: last frame */

       KASSERT(sav != NULL);
       KASSERT(sav->replay != NULL);

       replay = sav->replay;

       if (replay->wsize == 0)
               goto ok;        /* no need to check replay. */

       /* constant */
       frlast = replay->wsize - 1;
       wsizeb = replay->wsize << 3;

       /* sequence number of 0 is invalid */
       if (seq == 0)
               return 1;

       /* first time */
       if (replay->count == 0) {
               replay->lastseq = seq;
               memset(replay->bitmap, 0, replay->wsize);
               (replay->bitmap)[frlast] = 1;
               goto ok;
       }

       if (seq > replay->lastseq) {
               /* seq is larger than lastseq. */
               diff = seq - replay->lastseq;

               /* new larger sequence number */
               if (diff < wsizeb) {
                       /* In window */
                       /* set bit for this packet */
                       vshiftl(replay->bitmap, diff, replay->wsize);
                       (replay->bitmap)[frlast] |= 1;
               } else {
                       /* this packet has a "way larger" */
                       memset(replay->bitmap, 0, replay->wsize);
                       (replay->bitmap)[frlast] = 1;
               }
               replay->lastseq = seq;

               /* larger is good */
       } else {
               /* seq is equal or less than lastseq. */
               diff = replay->lastseq - seq;

               /* over range to check, i.e. too old or wrapped */
               if (diff >= wsizeb)
                       return 1;

               fr = frlast - diff / 8;

               /* this packet already seen ? */
               if ((replay->bitmap)[fr] & (1 << (diff % 8)))
                       return 1;

               /* mark as seen */
               (replay->bitmap)[fr] |= (1 << (diff % 8));

               /* out of order but good */
       }

ok:
       if (replay->count == ~0) {
               char buf[IPSEC_LOGSASTRLEN];

               /* set overflow flag */
               replay->overflow++;

               /* don't increment, no more packets accepted */
               if ((sav->flags & SADB_X_EXT_CYCSEQ) == 0)
                       return 1;

               IPSECLOG(LOG_WARNING, "replay counter made %d cycle. %s\n",
                   replay->overflow, ipsec_logsastr(sav, buf, sizeof(buf)));
       }

       replay->count++;

       return 0;
}

/*
* shift variable length buffer to left.
* IN:  bitmap: pointer to the buffer
*      nbit:   the number of to shift.
*      wsize:  buffer size (bytes).
*/
static void
vshiftl(unsigned char *bitmap, int nbit, int wsize)
{
       int s, j, i;
       unsigned char over;

       for (j = 0; j < nbit; j += 8) {
               s = (nbit - j < 8) ? (nbit - j): 8;
               bitmap[0] <<= s;
               for (i = 1; i < wsize; i++) {
                       over = (bitmap[i] >> (8 - s));
                       bitmap[i] <<= s;
                       bitmap[i-1] |= over;
               }
       }

       return;
}

/* Return a printable string for the address. */
const char *
ipsec_address(const union sockaddr_union *sa, char *buf, size_t size)
{
       switch (sa->sa.sa_family) {
       case AF_INET:
               in_print(buf, size, &sa->sin.sin_addr);
               return buf;
#if INET6
       case AF_INET6:
               in6_print(buf, size, &sa->sin6.sin6_addr);
               return buf;
#endif
       default:
               return "(unknown address family)";
       }
}

const char *
ipsec_logsastr(const struct secasvar *sav, char *buf, size_t size)
{
       const struct secasindex *saidx = &sav->sah->saidx;
       char sbuf[IPSEC_ADDRSTRLEN], dbuf[IPSEC_ADDRSTRLEN];

       KASSERTMSG(saidx->src.sa.sa_family == saidx->dst.sa.sa_family,
           "af family mismatch, src %u, dst %u",
           saidx->src.sa.sa_family, saidx->dst.sa.sa_family);

       snprintf(buf, size, "SA(SPI=%u src=%s dst=%s)",
           (u_int32_t)ntohl(sav->spi),
           ipsec_address(&saidx->src, sbuf, sizeof(sbuf)),
           ipsec_address(&saidx->dst, dbuf, sizeof(dbuf)));

       return buf;
}

#ifdef INET6
struct secpolicy *
ipsec6_check_policy(struct mbuf *m, struct inpcb *inp, int flags,
   int *needipsecp, int *errorp)
{
       struct secpolicy *sp = NULL;
       int error = 0;
       int needipsec = 0;

       if (ipsec_outdone(m)) {
               goto skippolicycheck;
       }
       if (inp && ipsec_pcb_skip_ipsec(inp->inp_sp, IPSEC_DIR_OUTBOUND)) {
               goto skippolicycheck;
       }
       sp = ipsec_checkpolicy(m, IPSEC_DIR_OUTBOUND, flags, &error, inp);

       /*
        * There are four return cases:
        *      sp != NULL                    apply IPsec policy
        *      sp == NULL, error == 0        no IPsec handling needed
        *      sp == NULL, error == -EINVAL  discard packet w/o error
        *      sp == NULL, error != 0        discard packet, report error
        */
       if (sp == NULL) {
               needipsec = 0;
       } else {
               needipsec = 1;
       }

skippolicycheck:
       *errorp = error;
       *needipsecp = needipsec;
       return sp;
}

/*
* calculate UDP checksum for UDP encapsulated ESP for IPv6.
*
* RFC2460(Internet Protocol, Version 6 Specification) says:
*
*   IPv6 receivers MUST discard UDP packets with a zero checksum.
*
* There is more relaxed specification RFC6935(IPv6 and UDP Checksums for
* Tunneled Packets). The document allows zero checksum. It's too
* late to publish, there are a lot of interoperability problems...
*/
void
ipsec6_udp_cksum(struct mbuf *m)
{
       struct ip6_hdr *ip6;
       uint16_t plen, uh_sum;
       int off;

       /* must called after m_pullup() */
       KASSERT(m->m_len >= sizeof(struct ip6_hdr));

       ip6 = mtod(m, struct ip6_hdr *);
       KASSERT(ip6->ip6_nxt == IPPROTO_UDP);

       /* ip6->ip6_plen can not be updated before ip6_output() */
       plen = m->m_pkthdr.len - sizeof(*ip6);
       KASSERT(plen >= sizeof(struct udphdr));

       uh_sum = in6_cksum(m, IPPROTO_UDP, sizeof(*ip6), plen);
       if (uh_sum == 0)
               uh_sum = 0xffff;

       off = sizeof(*ip6) + offsetof(struct udphdr, uh_sum);
       m_copyback(m, off, sizeof(uh_sum), (void *)&uh_sum);
}
#endif /* INET6 */

/*
* -----------------------------------------------------------------------------
*/

/* XXX this stuff doesn't belong here... */

static struct xformsw *xforms = NULL;

/*
* Register a transform; typically at system startup.
*/
void
xform_register(struct xformsw *xsp)
{
       xsp->xf_next = xforms;
       xforms = xsp;
}

/*
* Initialize transform support in an sav.
*/
int
xform_init(struct secasvar *sav, int xftype)
{
       struct xformsw *xsp;

       if (sav->tdb_xform != NULL)     /* previously initialized */
               return 0;
       for (xsp = xforms; xsp; xsp = xsp->xf_next)
               if (xsp->xf_type == xftype)
                       return (*xsp->xf_init)(sav, xsp);

       IPSECLOG(LOG_DEBUG, "no match for xform type %d\n", xftype);
       return EINVAL;
}

/*
* XXXJRT This should be done as a protosw init call.
*/
void
ipsec_attach(void)
{

       ipsec_output_init();

       ipsecstat_percpu = percpu_alloc(sizeof(uint64_t) * IPSEC_NSTATS);

       sysctl_net_inet_ipsec_setup(NULL);
#ifdef INET6
       sysctl_net_inet6_ipsec6_setup(NULL);
#endif

       ah_attach();
       esp_attach();
       ipcomp_attach();
       ipe4_attach();
#ifdef TCP_SIGNATURE
       tcpsignature_attach();
#endif
}