* Copyright (c) 2015 Internet Initiative Japan Inc.

/* $NetBSD: mvxpsec.c,v 1.20 2024/02/09 22:08:35 andvar Exp $ */
/*
* Copyright (c) 2015 Internet Initiative Japan Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#ifdef _KERNEL_OPT
#include "opt_ipsec.h"
#endif

/*
* Cryptographic Engine and Security Accelerator(MVXPSEC)
*/
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/conf.h>
#include <sys/proc.h>
#include <sys/bus.h>
#include <sys/evcnt.h>
#include <sys/device.h>
#include <sys/endian.h>
#include <sys/errno.h>
#include <sys/kmem.h>
#include <sys/mbuf.h>
#include <sys/callout.h>
#include <sys/pool.h>
#include <sys/cprng.h>
#include <sys/syslog.h>
#include <sys/mutex.h>
#include <sys/kthread.h>
#include <sys/atomic.h>
#include <sys/sha1.h>
#include <sys/md5.h>

#include <uvm/uvm_extern.h>

#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>

#include <net/net_stats.h>

#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>

#if NIPSEC > 0
#include <netipsec/esp_var.h>
#endif

#include <arm/cpufunc.h>
#include <arm/marvell/mvsocvar.h>
#include <arm/marvell/armadaxpreg.h>
#include <dev/marvell/marvellreg.h>
#include <dev/marvell/marvellvar.h>
#include <dev/marvell/mvxpsecreg.h>
#include <dev/marvell/mvxpsecvar.h>

#ifdef DEBUG
#define STATIC __attribute__ ((noinline)) extern
#define _STATIC __attribute__ ((noinline)) extern
#define INLINE __attribute__ ((noinline)) extern
#define _INLINE __attribute__ ((noinline)) extern
#else
#define STATIC static
#define _STATIC __attribute__ ((unused)) static
#define INLINE static inline
#define _INLINE __attribute__ ((unused)) static inline
#endif

/*
* IRQ and SRAM spaces for each of unit
* XXX: move to attach_args
*/
struct {
int err_int;
} mvxpsec_config[] = {
{ .err_int = ARMADAXP_IRQ_CESA0_ERR, }, /* unit 0 */
{ .err_int = ARMADAXP_IRQ_CESA1_ERR, }, /* unit 1 */
};
#define MVXPSEC_ERR_INT(sc) \
mvxpsec_config[device_unit((sc)->sc_dev)].err_int

/*
* AES
*/
#define MAXBC (128/32)
#define MAXKC (256/32)
#define MAXROUNDS 14
STATIC int mv_aes_ksched(uint8_t[4][MAXKC], int,
uint8_t[MAXROUNDS+1][4][MAXBC]);
STATIC int mv_aes_deckey(uint8_t *, uint8_t *, int);

/*
* device driver autoconf interface
*/
STATIC int mvxpsec_match(device_t, cfdata_t, void *);
STATIC void mvxpsec_attach(device_t, device_t, void *);
STATIC void mvxpsec_evcnt_attach(struct mvxpsec_softc *);

/*
* register setup
*/
STATIC int mvxpsec_wininit(struct mvxpsec_softc *, enum marvell_tags *);

/*
* timer(callout) interface
*
* XXX: callout is not MP safe...
*/
STATIC void mvxpsec_timer(void *);

/*
* interrupt interface
*/
STATIC int mvxpsec_intr(void *);
INLINE void mvxpsec_intr_cleanup(struct mvxpsec_softc *);
STATIC int mvxpsec_eintr(void *);
STATIC uint32_t mvxpsec_intr_ack(struct mvxpsec_softc *);
STATIC uint32_t mvxpsec_eintr_ack(struct mvxpsec_softc *);
INLINE void mvxpsec_intr_cnt(struct mvxpsec_softc *, int);

/*
* memory allocators and VM management
*/
STATIC struct mvxpsec_devmem *mvxpsec_alloc_devmem(struct mvxpsec_softc *,
paddr_t, int);
STATIC int mvxpsec_init_sram(struct mvxpsec_softc *);

/*
* Low-level DMA interface
*/
STATIC int mvxpsec_init_dma(struct mvxpsec_softc *,
struct marvell_attach_args *);
INLINE int mvxpsec_dma_wait(struct mvxpsec_softc *);
INLINE int mvxpsec_acc_wait(struct mvxpsec_softc *);
INLINE struct mvxpsec_descriptor_handle *mvxpsec_dma_getdesc(struct mvxpsec_softc *);
_INLINE void mvxpsec_dma_putdesc(struct mvxpsec_softc *, struct mvxpsec_descriptor_handle *);
INLINE void mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *,
uint32_t, uint32_t, uint32_t);
INLINE void mvxpsec_dma_cat(struct mvxpsec_softc *,
struct mvxpsec_descriptor_handle *, struct mvxpsec_descriptor_handle *);

/*
* High-level DMA interface
*/
INLINE int mvxpsec_dma_copy0(struct mvxpsec_softc *,
mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
INLINE int mvxpsec_dma_copy(struct mvxpsec_softc *,
mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t);
INLINE int mvxpsec_dma_acc_activate(struct mvxpsec_softc *,
mvxpsec_dma_ring *);
INLINE void mvxpsec_dma_finalize(struct mvxpsec_softc *,
mvxpsec_dma_ring *);
INLINE void mvxpsec_dma_free(struct mvxpsec_softc *,
mvxpsec_dma_ring *);
INLINE int mvxpsec_dma_copy_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);
INLINE int mvxpsec_dma_sync_packet(struct mvxpsec_softc *, struct mvxpsec_packet *);

/*
* Session management interface (OpenCrypto)
*/
#define MVXPSEC_SESSION(sid) ((sid) & 0x0fffffff)
#define MVXPSEC_SID(crd, sesn) (((crd) << 28) | ((sesn) & 0x0fffffff))
/* pool management */
STATIC int mvxpsec_session_ctor(void *, void *, int);
STATIC void mvxpsec_session_dtor(void *, void *);
STATIC int mvxpsec_packet_ctor(void *, void *, int);
STATIC void mvxpsec_packet_dtor(void *, void *);

/* session management */
STATIC struct mvxpsec_session *mvxpsec_session_alloc(struct mvxpsec_softc *);
STATIC void mvxpsec_session_dealloc(struct mvxpsec_session *);
INLINE struct mvxpsec_session *mvxpsec_session_lookup(struct mvxpsec_softc *, int);
INLINE int mvxpsec_session_ref(struct mvxpsec_session *);
INLINE void mvxpsec_session_unref(struct mvxpsec_session *);

/* packet management */
STATIC struct mvxpsec_packet *mvxpsec_packet_alloc(struct mvxpsec_session *);
INLINE void mvxpsec_packet_enqueue(struct mvxpsec_packet *);
STATIC void mvxpsec_packet_dealloc(struct mvxpsec_packet *);
STATIC int mvxpsec_done_packet(struct mvxpsec_packet *);

/* session header manegement */
STATIC int mvxpsec_header_finalize(struct mvxpsec_packet *);

/* packet queue management */
INLINE void mvxpsec_drop(struct mvxpsec_softc *, struct cryptop *, struct mvxpsec_packet *, int);
STATIC int mvxpsec_dispatch_queue(struct mvxpsec_softc *);

/* opencrypto operation */
INLINE int mvxpsec_parse_crd(struct mvxpsec_packet *, struct cryptodesc *);
INLINE int mvxpsec_parse_crp(struct mvxpsec_packet *);

/* payload data management */
INLINE int mvxpsec_packet_setcrp(struct mvxpsec_packet *, struct cryptop *);
STATIC int mvxpsec_packet_setdata(struct mvxpsec_packet *, void *, uint32_t);
STATIC int mvxpsec_packet_setmbuf(struct mvxpsec_packet *, struct mbuf *);
STATIC int mvxpsec_packet_setuio(struct mvxpsec_packet *, struct uio *);
STATIC int mvxpsec_packet_rdata(struct mvxpsec_packet *, int, int, void *);
_STATIC int mvxpsec_packet_wdata(struct mvxpsec_packet *, int, int, void *);
STATIC int mvxpsec_packet_write_iv(struct mvxpsec_packet *, void *, int);
STATIC int mvxpsec_packet_copy_iv(struct mvxpsec_packet *, int, int);

/* key pre-computation */
STATIC int mvxpsec_key_precomp(int, void *, int, void *, void *);
STATIC int mvxpsec_hmac_precomp(int, void *, int, void *, void *);

/* crypto operation management */
INLINE void mvxpsec_packet_reset_op(struct mvxpsec_packet *);
INLINE void mvxpsec_packet_update_op_order(struct mvxpsec_packet *, int);

/*
* parameter converters
*/
INLINE uint32_t mvxpsec_alg2acc(uint32_t alg);
INLINE uint32_t mvxpsec_aesklen(int klen);

/*
* string formatters
*/
_STATIC const char *s_ctrlreg(uint32_t);
_STATIC const char *s_winreg(uint32_t);
_STATIC const char *s_errreg(uint32_t);
_STATIC const char *s_xpsecintr(uint32_t);
_STATIC const char *s_ctlalg(uint32_t);
_STATIC const char *s_xpsec_op(uint32_t);
_STATIC const char *s_xpsec_enc(uint32_t);
_STATIC const char *s_xpsec_mac(uint32_t);
_STATIC const char *s_xpsec_frag(uint32_t);

/*
* debugging supports
*/
#ifdef MVXPSEC_DEBUG
_STATIC void mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *);
_STATIC void mvxpsec_dump_reg(struct mvxpsec_softc *);
_STATIC void mvxpsec_dump_sram(const char *, struct mvxpsec_softc *, size_t);
_STATIC void mvxpsec_dump_data(const char *, void *, size_t);

_STATIC void mvxpsec_dump_packet(const char *, struct mvxpsec_packet *);
_STATIC void mvxpsec_dump_packet_data(const char *, struct mvxpsec_packet *);
_STATIC void mvxpsec_dump_packet_desc(const char *, struct mvxpsec_packet *);

_STATIC void mvxpsec_dump_acc_config(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_encdata(const char *, uint32_t, uint32_t);
_STATIC void mvxpsec_dump_acc_enclen(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_enckey(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_enciv(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_macsrc(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_macdst(const char *, uint32_t);
_STATIC void mvxpsec_dump_acc_maciv(const char *, uint32_t);
#endif

/*
* global configurations, params, work spaces, ...
*
* XXX: use sysctl for global configurations
*/
/* waiting for device */
static int mvxpsec_wait_interval = 10; /* usec */
static int mvxpsec_wait_retry = 100; /* times = wait for 1 [msec] */
#ifdef MVXPSEC_DEBUG
static uint32_t mvxpsec_debug = MVXPSEC_DEBUG; /* debug level */
#endif

/*
* Register accessors
*/
#define MVXPSEC_WRITE(sc, off, val) \
bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (off), (val))
#define MVXPSEC_READ(sc, off) \
bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (off))

/*
* device driver autoconf interface
*/
CFATTACH_DECL2_NEW(mvxpsec_mbus, sizeof(struct mvxpsec_softc),
mvxpsec_match, mvxpsec_attach, NULL, NULL, NULL, NULL);

STATIC int
mvxpsec_match(device_t dev, cfdata_t match, void *aux)
{
struct marvell_attach_args *mva = aux;
uint32_t tag;
int window;

if (strcmp(mva->mva_name, match->cf_name) != 0)
return 0;
if (mva->mva_offset == MVA_OFFSET_DEFAULT)
return 0;

switch (mva->mva_unit) {
case 0:
tag = ARMADAXP_TAG_CRYPT0;
break;
case 1:
tag = ARMADAXP_TAG_CRYPT1;
break;
default:
aprint_error_dev(dev,
"unit %d is not supported\n", mva->mva_unit);
return 0;
}

window = mvsoc_target(tag, NULL, NULL, NULL, NULL);
if (window >= nwindow) {
aprint_error_dev(dev,
"Security Accelerator SRAM is not configured.\n");
return 0;
}

return 1;
}

STATIC void
mvxpsec_attach(device_t parent, device_t self, void *aux)
{
struct marvell_attach_args *mva = aux;
struct mvxpsec_softc *sc = device_private(self);
int v;
int i;

sc->sc_dev = self;

aprint_normal(": Marvell Crypto Engines and Security Accelerator\n");
aprint_naive("\n");
#ifdef MVXPSEC_MULTI_PACKET
aprint_normal_dev(sc->sc_dev, "multi-packet chained mode enabled.\n");
#else
aprint_normal_dev(sc->sc_dev, "multi-packet chained mode disabled.\n");
#endif
aprint_normal_dev(sc->sc_dev,
"Max %d sessions.\n", MVXPSEC_MAX_SESSIONS);

/* mutex */
mutex_init(&sc->sc_session_mtx, MUTEX_DEFAULT, IPL_NET);
mutex_init(&sc->sc_dma_mtx, MUTEX_DEFAULT, IPL_NET);
mutex_init(&sc->sc_queue_mtx, MUTEX_DEFAULT, IPL_NET);

/* Packet queue */
SIMPLEQ_INIT(&sc->sc_wait_queue);
SIMPLEQ_INIT(&sc->sc_run_queue);
SLIST_INIT(&sc->sc_free_list);
sc->sc_wait_qlen = 0;
#ifdef MVXPSEC_MULTI_PACKET
sc->sc_wait_qlimit = 16;
#else
sc->sc_wait_qlimit = 0;
#endif
sc->sc_free_qlen = 0;

/* Timer */
callout_init(&sc->sc_timeout, 0); /* XXX: use CALLOUT_MPSAFE */
callout_setfunc(&sc->sc_timeout, mvxpsec_timer, sc);

/* I/O */
sc->sc_iot = mva->mva_iot;
if (bus_space_subregion(mva->mva_iot, mva->mva_ioh,
mva->mva_offset, mva->mva_size, &sc->sc_ioh)) {
aprint_error_dev(self, "Cannot map registers\n");
return;
}

/* DMA */
sc->sc_dmat = mva->mva_dmat;
if (mvxpsec_init_dma(sc, mva) < 0)
return;

/* SRAM */
if (mvxpsec_init_sram(sc) < 0)
return;

/* Registers */
mvxpsec_wininit(sc, mva->mva_tags);

/* INTR */
MVXPSEC_WRITE(sc, MVXPSEC_INT_MASK, MVXPSEC_DEFAULT_INT);
MVXPSEC_WRITE(sc, MV_TDMA_ERR_MASK, MVXPSEC_DEFAULT_ERR);
sc->sc_done_ih =
marvell_intr_establish(mva->mva_irq, IPL_NET, mvxpsec_intr, sc);
/* XXX: should pass error IRQ using mva */
sc->sc_error_ih = marvell_intr_establish(MVXPSEC_ERR_INT(sc),
IPL_NET, mvxpsec_eintr, sc);
aprint_normal_dev(self,
"Error Reporting IRQ %d\n", MVXPSEC_ERR_INT(sc));

/* Initialize TDMA (It's enabled here, but waiting for SA) */
if (mvxpsec_dma_wait(sc) < 0)
panic("%s: DMA DEVICE not responding\n", __func__);
MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
v = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
v |= MV_TDMA_CONTROL_ENABLE;
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, v);

/* Initialize SA */
if (mvxpsec_acc_wait(sc) < 0)
panic("%s: MVXPSEC not responding\n", __func__);
v = MVXPSEC_READ(sc, MV_ACC_CONFIG);
v &= ~MV_ACC_CONFIG_STOP_ON_ERR;
v |= MV_ACC_CONFIG_MULT_PKT;
v |= MV_ACC_CONFIG_WAIT_TDMA;
v |= MV_ACC_CONFIG_ACT_TDMA;
MVXPSEC_WRITE(sc, MV_ACC_CONFIG, v);
MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);

/* Session */
sc->sc_session_pool =
pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
"mvxpsecpl", NULL, IPL_NET,
mvxpsec_session_ctor, mvxpsec_session_dtor, sc);
pool_cache_sethiwat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS);
pool_cache_setlowat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS / 2);
sc->sc_last_session = NULL;

/* Packet */
sc->sc_packet_pool =
pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0,
"mvxpsec_pktpl", NULL, IPL_NET,
mvxpsec_packet_ctor, mvxpsec_packet_dtor, sc);
pool_cache_sethiwat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS);
pool_cache_setlowat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS / 2);

/* Register to EVCNT framework */
mvxpsec_evcnt_attach(sc);

/* Register to Opencrypto */
for (i = 0; i < MVXPSEC_MAX_SESSIONS; i++) {
sc->sc_sessions[i] = NULL;
}
if (mvxpsec_register(sc))
panic("cannot initialize OpenCrypto module.\n");

return;
}

STATIC void
mvxpsec_evcnt_attach(struct mvxpsec_softc *sc)
{
struct mvxpsec_evcnt *sc_ev = &sc->sc_ev;

evcnt_attach_dynamic(&sc_ev->intr_all, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "Main Intr.");
evcnt_attach_dynamic(&sc_ev->intr_auth, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "Auth Intr.");
evcnt_attach_dynamic(&sc_ev->intr_des, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "DES Intr.");
evcnt_attach_dynamic(&sc_ev->intr_aes_enc, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "AES-Encrypt Intr.");
evcnt_attach_dynamic(&sc_ev->intr_aes_dec, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "AES-Decrypt Intr.");
evcnt_attach_dynamic(&sc_ev->intr_enc, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "Crypto Intr.");
evcnt_attach_dynamic(&sc_ev->intr_sa, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "SA Intr.");
evcnt_attach_dynamic(&sc_ev->intr_acctdma, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "AccTDMA Intr.");
evcnt_attach_dynamic(&sc_ev->intr_comp, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "TDMA-Complete Intr.");
evcnt_attach_dynamic(&sc_ev->intr_own, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "TDMA-Ownership Intr.");
evcnt_attach_dynamic(&sc_ev->intr_acctdma_cont, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "AccTDMA-Continue Intr.");

evcnt_attach_dynamic(&sc_ev->session_new, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "New-Session");
evcnt_attach_dynamic(&sc_ev->session_free, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Free-Session");

evcnt_attach_dynamic(&sc_ev->packet_ok, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Packet-OK");
evcnt_attach_dynamic(&sc_ev->packet_err, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Packet-ERR");

evcnt_attach_dynamic(&sc_ev->dispatch_packets, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Packet-Dispatch");
evcnt_attach_dynamic(&sc_ev->dispatch_queue, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Queue-Dispatch");
evcnt_attach_dynamic(&sc_ev->queue_full, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Queue-Full");
evcnt_attach_dynamic(&sc_ev->max_dispatch, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Max-Dispatch");
evcnt_attach_dynamic(&sc_ev->max_done, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Max-Done");
}

/*
* Register setup
*/
STATIC int mvxpsec_wininit(struct mvxpsec_softc *sc, enum marvell_tags *tags)
{
device_t pdev = device_parent(sc->sc_dev);
uint64_t base;
uint32_t size, reg;
int window, target, attr, rv, i;

/* disable all window */
for (window = 0; window < MV_TDMA_NWINDOW; window++)
{
MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), 0);
MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), 0);
}

for (window = 0, i = 0;
tags[i] != MARVELL_TAG_UNDEFINED && window < MV_TDMA_NWINDOW; i++) {
rv = marvell_winparams_by_tag(pdev, tags[i],
&target, &attr, &base, &size);
if (rv != 0 || size == 0)
continue;

if (base > 0xffffffffULL) {
aprint_error_dev(sc->sc_dev,
"can't remap window %d\n", window);
continue;
}

reg = MV_TDMA_BAR_BASE(base);
MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), reg);

reg = MV_TDMA_ATTR_TARGET(target);
reg |= MV_TDMA_ATTR_ATTR(attr);
reg |= MV_TDMA_ATTR_SIZE(size);
reg |= MV_TDMA_ATTR_ENABLE;
MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), reg);

window++;
}

return 0;
}

/*
* Timer handling
*/
STATIC void
mvxpsec_timer(void *aux)
{
struct mvxpsec_softc *sc = aux;
struct mvxpsec_packet *mv_p;
uint32_t reg;
int ndone;
int refill;
int s;

/* IPL_SOFTCLOCK */

log(LOG_ERR, "%s: device timeout.\n", __func__);
#ifdef MVXPSEC_DEBUG
mvxpsec_dump_reg(sc);
#endif

s = splnet();
/* stop security accelerator */
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP);

/* stop TDMA */
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, 0);

/* cleanup packet queue */
mutex_enter(&sc->sc_queue_mtx);
ndone = 0;
while ( (mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue)) != NULL) {
SIMPLEQ_REMOVE_HEAD(&sc->sc_run_queue, queue);

mv_p->crp->crp_etype = EINVAL;
mvxpsec_done_packet(mv_p);
ndone++;
}
MVXPSEC_EVCNT_MAX(sc, max_done, ndone);
sc->sc_flags &= ~HW_RUNNING;
refill = (sc->sc_wait_qlen > 0) ? 1 : 0;
mutex_exit(&sc->sc_queue_mtx);

/* reenable TDMA */
if (mvxpsec_dma_wait(sc) < 0)
panic("%s: failed to reset DMA DEVICE. give up.", __func__);
MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0);
MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0);
MVXPSEC_WRITE(sc, MV_TDMA_DST, 0);
MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0);
MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0);
reg = MV_TDMA_DEFAULT_CONTROL;
reg |= MV_TDMA_CONTROL_ENABLE;
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, reg);

if (mvxpsec_acc_wait(sc) < 0)
panic("%s: failed to reset MVXPSEC. give up.", __func__);
reg = MV_ACC_CONFIG_MULT_PKT;
reg |= MV_ACC_CONFIG_WAIT_TDMA;
reg |= MV_ACC_CONFIG_ACT_TDMA;
MVXPSEC_WRITE(sc, MV_ACC_CONFIG, reg);
MVXPSEC_WRITE(sc, MV_ACC_DESC, 0);

if (refill) {
mutex_enter(&sc->sc_queue_mtx);
mvxpsec_dispatch_queue(sc);
mutex_exit(&sc->sc_queue_mtx);
}

crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
splx(s);
}

/*
* DMA handling
*/

/*
* Allocate kernel devmem and DMA safe memory with bus_dma API
* used for DMA descriptors.
*
* if phys != 0, assume phys is a DMA safe memory and bypass
* allocator.
*/
STATIC struct mvxpsec_devmem *
mvxpsec_alloc_devmem(struct mvxpsec_softc *sc, paddr_t phys, int size)
{
struct mvxpsec_devmem *devmem;
bus_dma_segment_t seg;
int rseg;
int err;

if (sc == NULL)
return NULL;

devmem = kmem_alloc(sizeof(*devmem), KM_SLEEP);
devmem->size = size;

if (phys) {
seg.ds_addr = phys;
seg.ds_len = devmem->size;
rseg = 1;
err = 0;
}
else {
err = bus_dmamem_alloc(sc->sc_dmat,
devmem->size, PAGE_SIZE, 0,
&seg, MVXPSEC_DMA_MAX_SEGS, &rseg, BUS_DMA_NOWAIT);
}
if (err) {
aprint_error_dev(sc->sc_dev, "can't alloc DMA buffer\n");
goto fail_kmem_free;
}

err = bus_dmamem_map(sc->sc_dmat, &seg, rseg,
devmem->size, &devmem->kva, BUS_DMA_NOWAIT);
if (err) {
aprint_error_dev(sc->sc_dev, "can't map DMA buffer\n");
goto fail_dmamem_free;
}

err = bus_dmamap_create(sc->sc_dmat,
size, 1, size, 0, BUS_DMA_NOWAIT, &devmem->map);
if (err) {
aprint_error_dev(sc->sc_dev, "can't create DMA map\n");
goto fail_unmap;
}

err = bus_dmamap_load(sc->sc_dmat,
devmem->map, devmem->kva, devmem->size, NULL,
BUS_DMA_NOWAIT);
if (err) {
aprint_error_dev(sc->sc_dev,
"can't load DMA buffer VA:%p PA:0x%08x\n",
devmem->kva, (int)seg.ds_addr);
goto fail_destroy;
}

return devmem;

fail_destroy:
bus_dmamap_destroy(sc->sc_dmat, devmem->map);
fail_unmap:
bus_dmamem_unmap(sc->sc_dmat, devmem->kva, devmem->size);
fail_dmamem_free:
bus_dmamem_free(sc->sc_dmat, &seg, rseg);
fail_kmem_free:
kmem_free(devmem, sizeof(*devmem));

return NULL;
}

/*
* Get DMA Descriptor from (DMA safe) descriptor pool.
*/
INLINE struct mvxpsec_descriptor_handle *
mvxpsec_dma_getdesc(struct mvxpsec_softc *sc)
{
struct mvxpsec_descriptor_handle *entry;

/* must called with sc->sc_dma_mtx held */
KASSERT(mutex_owned(&sc->sc_dma_mtx));

if (sc->sc_desc_ring_prod == sc->sc_desc_ring_cons)
return NULL;

entry = &sc->sc_desc_ring[sc->sc_desc_ring_prod];
sc->sc_desc_ring_prod++;
if (sc->sc_desc_ring_prod >= sc->sc_desc_ring_size)
sc->sc_desc_ring_prod -= sc->sc_desc_ring_size;

return entry;
}

/*
* Put DMA Descriptor to descriptor pool.
*/
_INLINE void
mvxpsec_dma_putdesc(struct mvxpsec_softc *sc,
struct mvxpsec_descriptor_handle *dh)
{
/* must called with sc->sc_dma_mtx held */
KASSERT(mutex_owned(&sc->sc_dma_mtx));

sc->sc_desc_ring_cons++;
if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;

return;
}

/*
* Setup DMA Descriptor
* copy from 'src' to 'dst' by 'size' bytes.
* 'src' or 'dst' must be SRAM address.
*/
INLINE void
mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *dh,
uint32_t dst, uint32_t src, uint32_t size)
{
struct mvxpsec_descriptor *desc;

desc = (struct mvxpsec_descriptor *)dh->_desc;

desc->tdma_dst = dst;
desc->tdma_src = src;
desc->tdma_word0 = size;
if (size != 0)
desc->tdma_word0 |= MV_TDMA_CNT_OWN;
/* size == 0 is owned by ACC, not TDMA */

#ifdef MVXPSEC_DEBUG
mvxpsec_dump_dmaq(dh);
#endif
}

/*
* Concat 2 DMA
*/
INLINE void
mvxpsec_dma_cat(struct mvxpsec_softc *sc,
struct mvxpsec_descriptor_handle *dh1,
struct mvxpsec_descriptor_handle *dh2)
{
((struct mvxpsec_descriptor*)dh1->_desc)->tdma_nxt = dh2->phys_addr;
MVXPSEC_SYNC_DESC(sc, dh1, BUS_DMASYNC_PREWRITE);
}

/*
* Schedule DMA Copy
*/
INLINE int
mvxpsec_dma_copy0(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
uint32_t dst, uint32_t src, uint32_t size)
{
struct mvxpsec_descriptor_handle *dh;

dh = mvxpsec_dma_getdesc(sc);
if (dh == NULL) {
log(LOG_ERR, "%s: descriptor full\n", __func__);
return -1;
}

mvxpsec_dma_setup(dh, dst, src, size);
if (r->dma_head == NULL) {
r->dma_head = dh;
r->dma_last = dh;
r->dma_size = 1;
}
else {
mvxpsec_dma_cat(sc, r->dma_last, dh);
r->dma_last = dh;
r->dma_size++;
}

return 0;
}

INLINE int
mvxpsec_dma_copy(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r,
uint32_t dst, uint32_t src, uint32_t size)
{
if (size == 0) /* 0 is very special descriptor */
return 0;

return mvxpsec_dma_copy0(sc, r, dst, src, size);
}

/*
* Schedule ACC Activate
*/
INLINE int
mvxpsec_dma_acc_activate(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
{
return mvxpsec_dma_copy0(sc, r, 0, 0, 0);
}

/*
* Finalize DMA setup
*/
INLINE void
mvxpsec_dma_finalize(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
{
struct mvxpsec_descriptor_handle *dh;

dh = r->dma_last;
((struct mvxpsec_descriptor*)dh->_desc)->tdma_nxt = 0;
MVXPSEC_SYNC_DESC(sc, dh, BUS_DMASYNC_PREWRITE);
}

/*
* Free entire DMA ring
*/
INLINE void
mvxpsec_dma_free(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r)
{
sc->sc_desc_ring_cons += r->dma_size;
if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size)
sc->sc_desc_ring_cons -= sc->sc_desc_ring_size;
r->dma_head = NULL;
r->dma_last = NULL;
r->dma_size = 0;
}

/*
* create DMA descriptor chain for the packet
*/
INLINE int
mvxpsec_dma_copy_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
uint32_t src, dst, len;
uint32_t pkt_off, pkt_off_r;
int err;
int i;

/* must called with sc->sc_dma_mtx held */
KASSERT(mutex_owned(&sc->sc_dma_mtx));

/*
* set offset for mem->device copy
*
* typical packet image:
*
* enc_ivoff
* mac_off
* |
* | enc_off
* | |
* v v
* +----+--------...
* |IV |DATA
* +----+--------...
*/
pkt_off = 0;
if (mv_p->mac_off > 0)
pkt_off = mv_p->mac_off;
if ((mv_p->flags & CRP_EXT_IV) == 0 && pkt_off > mv_p->enc_ivoff)
pkt_off = mv_p->enc_ivoff;
if (mv_p->enc_off > 0 && pkt_off > mv_p->enc_off)
pkt_off = mv_p->enc_off;
pkt_off_r = pkt_off;

/* make DMA descriptors to copy packet header: DRAM -> SRAM */
dst = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
src = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
len = sizeof(mv_p->pkt_header);
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
if (__predict_false(err))
return err;

/*
* make DMA descriptors to copy session header: DRAM -> SRAM
* we can reuse session header on SRAM if session is not changed.
*/
if (sc->sc_last_session != mv_s) {
dst = (uint32_t)MVXPSEC_SRAM_SESS_HDR_PA(sc);
src = (uint32_t)mv_s->session_header_map->dm_segs[0].ds_addr;
len = sizeof(mv_s->session_header);
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
if (__predict_false(err))
return err;
sc->sc_last_session = mv_s;
}

/* make DMA descriptor to copy payload data: DRAM -> SRAM */
dst = MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
src = mv_p->data_map->dm_segs[i].ds_addr;
len = mv_p->data_map->dm_segs[i].ds_len;
if (pkt_off) {
if (len <= pkt_off) {
/* ignore the segment */
dst += len;
pkt_off -= len;
continue;
}
/* copy from the middle of the segment */
dst += pkt_off;
src += pkt_off;
len -= pkt_off;
pkt_off = 0;
}
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
if (__predict_false(err))
return err;
dst += len;
}

/* make special descriptor to activate security accelerator */
err = mvxpsec_dma_acc_activate(sc, &mv_p->dma_ring);
if (__predict_false(err))
return err;

/* make DMA descriptors to copy payload: SRAM -> DRAM */
src = (uint32_t)MVXPSEC_SRAM_PAYLOAD_PA(sc, 0);
for (i = 0; i < mv_p->data_map->dm_nsegs; i++) {
dst = (uint32_t)mv_p->data_map->dm_segs[i].ds_addr;
len = (uint32_t)mv_p->data_map->dm_segs[i].ds_len;
if (pkt_off_r) {
if (len <= pkt_off_r) {
/* ignore the segment */
src += len;
pkt_off_r -= len;
continue;
}
/* copy from the middle of the segment */
src += pkt_off_r;
dst += pkt_off_r;
len -= pkt_off_r;
pkt_off_r = 0;
}
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
if (__predict_false(err))
return err;
src += len;
}
KASSERT(pkt_off == 0);
KASSERT(pkt_off_r == 0);

/*
* make DMA descriptors to copy packet header: SRAM->DRAM
* if IV is present in the payload, no need to copy.
*/
if (mv_p->flags & CRP_EXT_IV) {
dst = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr;
src = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc);
len = sizeof(mv_p->pkt_header);
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len);
if (__predict_false(err))
return err;
}

return 0;
}

INLINE int
mvxpsec_dma_sync_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p)
{
/* sync packet header */
bus_dmamap_sync(sc->sc_dmat,
mv_p->pkt_header_map, 0, sizeof(mv_p->pkt_header),
BUS_DMASYNC_PREWRITE);

#ifdef MVXPSEC_DEBUG
/* sync session header */
if (mvxpsec_debug != 0) {
struct mvxpsec_session *mv_s = mv_p->mv_s;

/* only debug code touch the session header after newsession */
bus_dmamap_sync(sc->sc_dmat,
mv_s->session_header_map,
0, sizeof(mv_s->session_header),
BUS_DMASYNC_PREWRITE);
}
#endif

/* sync packet buffer */
bus_dmamap_sync(sc->sc_dmat,
mv_p->data_map, 0, mv_p->data_len,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

return 0;
}

/*
* Initialize MVXPSEC Internal SRAM
*
* - must be called after DMA initialization.
* - make VM mapping for SRAM area on MBus.
*/
STATIC int
mvxpsec_init_sram(struct mvxpsec_softc *sc)
{
uint32_t tag, target, attr, base, size;
vaddr_t va;
int window;

switch (device_unit(sc->sc_dev)) {
case 0:
tag = ARMADAXP_TAG_CRYPT0;
break;
case 1:
tag = ARMADAXP_TAG_CRYPT1;
break;
default:
aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
return -1;
}

window = mvsoc_target(tag, &target, &attr, &base, &size);
if (window >= nwindow) {
aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n");
return -1;
}

if (sizeof(struct mvxpsec_crypt_sram) > size) {
aprint_error_dev(sc->sc_dev,
"SRAM Data Structure Excceeds SRAM window size.\n");
return -1;
}

aprint_normal_dev(sc->sc_dev,
"internal SRAM window at 0x%08x-0x%08x",
base, base + size - 1);
sc->sc_sram_pa = base;

/* get vmspace to read/write device internal SRAM */
va = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE,
UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
if (va == 0) {
aprint_error_dev(sc->sc_dev, "cannot map SRAM window\n");
sc->sc_sram_va = NULL;
aprint_normal("\n");
return 0;
}
/* XXX: not working. PMAP_NOCACHE is not affected? */
pmap_kenter_pa(va, base, VM_PROT_READ|VM_PROT_WRITE, PMAP_NOCACHE);
pmap_update(pmap_kernel());
sc->sc_sram_va = (void *)va;
aprint_normal(" va %p\n", sc->sc_sram_va);
memset(sc->sc_sram_va, 0xff, MV_ACC_SRAM_SIZE);

return 0;
}

/*
* Initialize TDMA engine.
*/
STATIC int
mvxpsec_init_dma(struct mvxpsec_softc *sc, struct marvell_attach_args *mva)
{
struct mvxpsec_descriptor_handle *dh;
uint8_t *va;
paddr_t pa;
off_t va_off, pa_off;
int i, n, seg, ndh;

/* Init Deviced's control parameters (disabled yet) */
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, MV_TDMA_DEFAULT_CONTROL);

/* Init Software DMA Handlers */
sc->sc_devmem_desc =
mvxpsec_alloc_devmem(sc, 0, PAGE_SIZE * MVXPSEC_DMA_DESC_PAGES);
ndh = (PAGE_SIZE / sizeof(struct mvxpsec_descriptor))
* MVXPSEC_DMA_DESC_PAGES;
sc->sc_desc_ring =
kmem_alloc(sizeof(struct mvxpsec_descriptor_handle) * ndh,
KM_SLEEP);
aprint_normal_dev(sc->sc_dev, "%d DMA handles in %zu bytes array\n",
ndh, sizeof(struct mvxpsec_descriptor_handle) * ndh);

ndh = 0;
for (seg = 0; seg < devmem_nseg(sc->sc_devmem_desc); seg++) {
va = devmem_va(sc->sc_devmem_desc);
pa = devmem_pa(sc->sc_devmem_desc, seg);
n = devmem_palen(sc->sc_devmem_desc, seg) /
sizeof(struct mvxpsec_descriptor);
va_off = (PAGE_SIZE * seg);
pa_off = 0;
for (i = 0; i < n; i++) {
dh = &sc->sc_desc_ring[ndh];
dh->map = devmem_map(sc->sc_devmem_desc);
dh->off = va_off + pa_off;
dh->_desc = (void *)(va + va_off + pa_off);
dh->phys_addr = pa + pa_off;
pa_off += sizeof(struct mvxpsec_descriptor);
ndh++;
}
}
sc->sc_desc_ring_size = ndh;
sc->sc_desc_ring_prod = 0;
sc->sc_desc_ring_cons = sc->sc_desc_ring_size - 1;

return 0;
}

/*
* Wait for TDMA controller become idle
*/
INLINE int
mvxpsec_dma_wait(struct mvxpsec_softc *sc)
{
int retry = 0;

while (MVXPSEC_READ(sc, MV_TDMA_CONTROL) & MV_TDMA_CONTROL_ACT) {
delay(mvxpsec_wait_interval);
if (retry++ >= mvxpsec_wait_retry)
return -1;
}
return 0;
}

/*
* Wait for Security Accelerator become idle
*/
INLINE int
mvxpsec_acc_wait(struct mvxpsec_softc *sc)
{
int retry = 0;

while (MVXPSEC_READ(sc, MV_ACC_COMMAND) & MV_ACC_COMMAND_ACT) {
delay(mvxpsec_wait_interval);
if (++retry >= mvxpsec_wait_retry)
return -1;
}
return 0;
}

/*
* Entry of interrupt handler
*
* register this to kernel via marvell_intr_establish()
*/
int
mvxpsec_intr(void *arg)
{
struct mvxpsec_softc *sc = arg;
uint32_t v;

/* IPL_NET */
while ((v = mvxpsec_intr_ack(sc)) != 0) {
mvxpsec_intr_cnt(sc, v);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "MVXPSEC Intr 0x%08x\n", v);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "%s\n", s_xpsecintr(v));
#ifdef MVXPSEC_DEBUG
mvxpsec_dump_reg(sc);
#endif

/* call high-level handlers */
if (v & MVXPSEC_INT_ACCTDMA)
mvxpsec_done(sc);
}

return 0;
}

INLINE void
mvxpsec_intr_cleanup(struct mvxpsec_softc *sc)
{
struct mvxpsec_packet *mv_p;

/* must called with sc->sc_dma_mtx held */
KASSERT(mutex_owned(&sc->sc_dma_mtx));

/*
* there is only one intr for run_queue.
* no one touch sc_run_queue.
*/
SIMPLEQ_FOREACH(mv_p, &sc->sc_run_queue, queue)
mvxpsec_dma_free(sc, &mv_p->dma_ring);
}

/*
* Acknowledge to interrupt
*
* read cause bits, clear it, and return it.
* NOTE: multiple cause bits may be returned at once.
*/
STATIC uint32_t
mvxpsec_intr_ack(struct mvxpsec_softc *sc)
{
uint32_t reg;

reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
reg &= MVXPSEC_DEFAULT_INT;
MVXPSEC_WRITE(sc, MVXPSEC_INT_CAUSE, ~reg);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));

return reg;
}

/*
* Entry of TDMA error interrupt handler
*
* register this to kernel via marvell_intr_establish()
*/
int
mvxpsec_eintr(void *arg)
{
struct mvxpsec_softc *sc = arg;
uint32_t err;

/* IPL_NET */
again:
err = mvxpsec_eintr_ack(sc);
if (err == 0)
goto done;

log(LOG_ERR, "%s: DMA Error Interrupt: %s\n", __func__,
s_errreg(err));
#ifdef MVXPSEC_DEBUG
mvxpsec_dump_reg(sc);
#endif

goto again;
done:
return 0;
}

/*
* Acknowledge to TDMA error interrupt
*
* read cause bits, clear it, and return it.
* NOTE: multiple cause bits may be returned at once.
*/
STATIC uint32_t
mvxpsec_eintr_ack(struct mvxpsec_softc *sc)
{
uint32_t reg;

reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
reg &= MVXPSEC_DEFAULT_ERR;
MVXPSEC_WRITE(sc, MV_TDMA_ERR_CAUSE, ~reg);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg));

return reg;
}

/*
* Interrupt statistics
*
* this is NOT a statistics of how many times the events 'occurred'.
* this ONLY means how many times the events 'handled'.
*/
INLINE void
mvxpsec_intr_cnt(struct mvxpsec_softc *sc, int cause)
{
MVXPSEC_EVCNT_INCR(sc, intr_all);
if (cause & MVXPSEC_INT_AUTH)
MVXPSEC_EVCNT_INCR(sc, intr_auth);
if (cause & MVXPSEC_INT_DES)
MVXPSEC_EVCNT_INCR(sc, intr_des);
if (cause & MVXPSEC_INT_AES_ENC)
MVXPSEC_EVCNT_INCR(sc, intr_aes_enc);
if (cause & MVXPSEC_INT_AES_DEC)
MVXPSEC_EVCNT_INCR(sc, intr_aes_dec);
if (cause & MVXPSEC_INT_ENC)
MVXPSEC_EVCNT_INCR(sc, intr_enc);
if (cause & MVXPSEC_INT_SA)
MVXPSEC_EVCNT_INCR(sc, intr_sa);
if (cause & MVXPSEC_INT_ACCTDMA)
MVXPSEC_EVCNT_INCR(sc, intr_acctdma);
if (cause & MVXPSEC_INT_TDMA_COMP)
MVXPSEC_EVCNT_INCR(sc, intr_comp);
if (cause & MVXPSEC_INT_TDMA_OWN)
MVXPSEC_EVCNT_INCR(sc, intr_own);
if (cause & MVXPSEC_INT_ACCTDMA_CONT)
MVXPSEC_EVCNT_INCR(sc, intr_acctdma_cont);
}

/*
* Setup MVXPSEC header structure.
*
* the header contains descriptor of security accelerator,
* key material of ciphers, iv of ciphers and macs, ...
*
* the header is transferred to MVXPSEC Internal SRAM by TDMA,
* and parsed by MVXPSEC H/W.
*/
STATIC int
mvxpsec_header_finalize(struct mvxpsec_packet *mv_p)
{
struct mvxpsec_acc_descriptor *desc = &mv_p->pkt_header.desc;
int enc_start, enc_len, iv_offset;
int mac_start, mac_len, mac_offset;

/* offset -> device address */
enc_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_off);
enc_len = mv_p->enc_len;
if (mv_p->flags & CRP_EXT_IV)
iv_offset = mv_p->enc_ivoff;
else
iv_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_ivoff);
mac_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_off);
mac_len = mv_p->mac_len;
mac_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_dst);

MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"PAYLOAD at 0x%08x\n", (int)MVXPSEC_SRAM_PAYLOAD_OFF);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"ENC from 0x%08x\n", enc_start);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"MAC from 0x%08x\n", mac_start);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"MAC to 0x%08x\n", mac_offset);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"ENC IV at 0x%08x\n", iv_offset);

/* setup device addresses in Security Accelerator Descriptors */
desc->acc_encdata = MV_ACC_DESC_ENC_DATA(enc_start, enc_start);
desc->acc_enclen = MV_ACC_DESC_ENC_LEN(enc_len);
if (desc->acc_config & MV_ACC_CRYPTO_DECRYPT)
desc->acc_enckey =
MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_D_DA);
else
desc->acc_enckey =
MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_DA);
desc->acc_enciv =
MV_ACC_DESC_ENC_IV(MVXPSEC_SRAM_IV_WORK_DA, iv_offset);

desc->acc_macsrc = MV_ACC_DESC_MAC_SRC(mac_start, mac_len);
desc->acc_macdst = MV_ACC_DESC_MAC_DST(mac_offset, mac_len);
desc->acc_maciv =
MV_ACC_DESC_MAC_IV(MVXPSEC_SRAM_MIV_IN_DA,
MVXPSEC_SRAM_MIV_OUT_DA);

return 0;
}

/*
* constractor of session structure.
*
* this constrator will be called by pool_cache framework.
*/
STATIC int
mvxpsec_session_ctor(void *arg, void *obj, int flags)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_session *mv_s = obj;

/* pool is owned by softc */
mv_s->sc = sc;

/* Create and load DMA map for session header */
mv_s->session_header_map = 0;
if (bus_dmamap_create(sc->sc_dmat,
sizeof(mv_s->session_header), 1,
sizeof(mv_s->session_header), 0,
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
&mv_s->session_header_map)) {
log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
goto fail;
}
if (bus_dmamap_load(sc->sc_dmat, mv_s->session_header_map,
&mv_s->session_header, sizeof(mv_s->session_header),
NULL, BUS_DMA_NOWAIT)) {
log(LOG_ERR, "%s: cannot load header\n", __func__);
goto fail;
}

return 0;
fail:
if (mv_s->session_header_map)
bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
return ENOMEM;
}

/*
* destractor of session structure.
*
* this destrator will be called by pool_cache framework.
*/
STATIC void
mvxpsec_session_dtor(void *arg, void *obj)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_session *mv_s = obj;

if (mv_s->sc != sc)
panic("inconsitent context\n");

bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map);
}

/*
* constructor of packet structure.
*/
STATIC int
mvxpsec_packet_ctor(void *arg, void *obj, int flags)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_packet *mv_p = obj;

mv_p->dma_ring.dma_head = NULL;
mv_p->dma_ring.dma_last = NULL;
mv_p->dma_ring.dma_size = 0;

/* Create and load DMA map for packet header */
mv_p->pkt_header_map = 0;
if (bus_dmamap_create(sc->sc_dmat,
sizeof(mv_p->pkt_header), 1, sizeof(mv_p->pkt_header), 0,
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
&mv_p->pkt_header_map)) {
log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
goto fail;
}
if (bus_dmamap_load(sc->sc_dmat, mv_p->pkt_header_map,
&mv_p->pkt_header, sizeof(mv_p->pkt_header),
NULL, BUS_DMA_NOWAIT)) {
log(LOG_ERR, "%s: cannot load header\n", __func__);
goto fail;
}

/* Create DMA map for session data. */
mv_p->data_map = 0;
if (bus_dmamap_create(sc->sc_dmat,
MVXPSEC_DMA_MAX_SIZE, MVXPSEC_DMA_MAX_SEGS, MVXPSEC_DMA_MAX_SIZE,
0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mv_p->data_map)) {
log(LOG_ERR, "%s: cannot create DMA map\n", __func__);
goto fail;
}

return 0;
fail:
if (mv_p->pkt_header_map)
bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
if (mv_p->data_map)
bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
return ENOMEM;
}

/*
* destractor of packet structure.
*/
STATIC void
mvxpsec_packet_dtor(void *arg, void *obj)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_packet *mv_p = obj;

mutex_enter(&sc->sc_dma_mtx);
mvxpsec_dma_free(sc, &mv_p->dma_ring);
mutex_exit(&sc->sc_dma_mtx);
bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map);
bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map);
}

/*
* allocate new session structure.
*/
STATIC struct mvxpsec_session *
mvxpsec_session_alloc(struct mvxpsec_softc *sc)
{
struct mvxpsec_session *mv_s;

mv_s = pool_cache_get(sc->sc_session_pool, PR_NOWAIT);
if (mv_s == NULL) {
log(LOG_ERR, "%s: cannot allocate memory\n", __func__);
return NULL;
}
mv_s->refs = 1; /* 0 means session is already invalid */
mv_s->sflags = 0;

return mv_s;
}

/*
* deallocate session structure.
*/
STATIC void
mvxpsec_session_dealloc(struct mvxpsec_session *mv_s)
{
struct mvxpsec_softc *sc = mv_s->sc;

mv_s->sflags |= DELETED;
mvxpsec_session_unref(mv_s);
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);

return;
}

STATIC int
mvxpsec_session_ref(struct mvxpsec_session *mv_s)
{
uint32_t refs;

if (mv_s->sflags & DELETED) {
log(LOG_ERR,
"%s: session is already deleted.\n", __func__);
return -1;
}

refs = atomic_inc_32_nv(&mv_s->refs);
if (refs == 1) {
/*
* a session with refs == 0 is
* already invalidated. revert it.
* XXX: use CAS ?
*/
atomic_dec_32(&mv_s->refs);
log(LOG_ERR,
"%s: session is already invalidated.\n", __func__);
return -1;
}

return 0;
}

STATIC void
mvxpsec_session_unref(struct mvxpsec_session *mv_s)
{
uint32_t refs;

membar_release();
refs = atomic_dec_32_nv(&mv_s->refs);
if (refs == 0) {
membar_acquire();
pool_cache_put(mv_s->sc->sc_session_pool, mv_s);
}
}

/*
* look for session is exist or not
*/
INLINE struct mvxpsec_session *
mvxpsec_session_lookup(struct mvxpsec_softc *sc, int sid)
{
struct mvxpsec_session *mv_s;
int session;

/* must called sc->sc_session_mtx held */
KASSERT(mutex_owned(&sc->sc_session_mtx));

session = MVXPSEC_SESSION(sid);
if (__predict_false(session > MVXPSEC_MAX_SESSIONS)) {
log(LOG_ERR, "%s: session number too large %d\n",
__func__, session);
return NULL;
}
if (__predict_false( (mv_s = sc->sc_sessions[session]) == NULL)) {
log(LOG_ERR, "%s: invalid session %d\n",
__func__, session);
return NULL;
}

KASSERT(mv_s->sid == session);

return mv_s;
}

/*
* allocation new packet structure.
*/
STATIC struct mvxpsec_packet *
mvxpsec_packet_alloc(struct mvxpsec_session *mv_s)
{
struct mvxpsec_softc *sc = mv_s->sc;
struct mvxpsec_packet *mv_p;

/* must be called mv_queue_mtx held. */
KASSERT(mutex_owned(&sc->sc_queue_mtx));
/* must be called mv_session_mtx held. */
KASSERT(mutex_owned(&sc->sc_session_mtx));

if (mvxpsec_session_ref(mv_s) < 0) {
log(LOG_ERR, "%s: invalid session.\n", __func__);
return NULL;
}

if ( (mv_p = SLIST_FIRST(&sc->sc_free_list)) != NULL) {
SLIST_REMOVE_HEAD(&sc->sc_free_list, free_list);
sc->sc_free_qlen--;
}
else {
mv_p = pool_cache_get(sc->sc_packet_pool, PR_NOWAIT);
if (mv_p == NULL) {
log(LOG_ERR, "%s: cannot allocate memory\n",
__func__);
mvxpsec_session_unref(mv_s);
return NULL;
}
}
mv_p->mv_s = mv_s;
mv_p->flags = 0;
mv_p->data_ptr = NULL;

return mv_p;
}

/*
* free packet structure.
*/
STATIC void
mvxpsec_packet_dealloc(struct mvxpsec_packet *mv_p)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
struct mvxpsec_softc *sc = mv_s->sc;

/* must called with sc->sc_queue_mtx held */
KASSERT(mutex_owned(&sc->sc_queue_mtx));

if (mv_p->dma_ring.dma_size != 0) {
sc->sc_desc_ring_cons += mv_p->dma_ring.dma_size;
}
mv_p->dma_ring.dma_head = NULL;
mv_p->dma_ring.dma_last = NULL;
mv_p->dma_ring.dma_size = 0;

if (mv_p->data_map) {
if (mv_p->flags & RDY_DATA) {
bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
mv_p->flags &= ~RDY_DATA;
}
}

if (sc->sc_free_qlen > sc->sc_wait_qlimit)
pool_cache_put(sc->sc_packet_pool, mv_p);
else {
SLIST_INSERT_HEAD(&sc->sc_free_list, mv_p, free_list);
sc->sc_free_qlen++;
}
mvxpsec_session_unref(mv_s);
}

INLINE void
mvxpsec_packet_enqueue(struct mvxpsec_packet *mv_p)
{
struct mvxpsec_softc *sc = mv_p->mv_s->sc;
struct mvxpsec_packet *last_packet;
struct mvxpsec_descriptor_handle *cur_dma, *prev_dma;

/* must called with sc->sc_queue_mtx held */
KASSERT(mutex_owned(&sc->sc_queue_mtx));

if (sc->sc_wait_qlen == 0) {
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
sc->sc_wait_qlen++;
mv_p->flags |= SETUP_DONE;
return;
}

last_packet = SIMPLEQ_LAST(&sc->sc_wait_queue, mvxpsec_packet, queue);
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
sc->sc_wait_qlen++;

/* chain the DMA */
cur_dma = mv_p->dma_ring.dma_head;
prev_dma = last_packet->dma_ring.dma_last;
mvxpsec_dma_cat(sc, prev_dma, cur_dma);
mv_p->flags |= SETUP_DONE;
}

/*
* called by interrupt handler
*/
STATIC int
mvxpsec_done_packet(struct mvxpsec_packet *mv_p)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
struct mvxpsec_softc *sc = mv_s->sc;

KASSERT((mv_p->flags & RDY_DATA));
KASSERT((mv_p->flags & SETUP_DONE));

/* unload data */
bus_dmamap_sync(sc->sc_dmat, mv_p->data_map,
0, mv_p->data_len,
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, mv_p->data_map);
mv_p->flags &= ~RDY_DATA;

#ifdef MVXPSEC_DEBUG
if (mvxpsec_debug != 0) {
int s;

bus_dmamap_sync(sc->sc_dmat, mv_p->pkt_header_map,
0, sizeof(mv_p->pkt_header),
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
bus_dmamap_sync(sc->sc_dmat, mv_s->session_header_map,
0, sizeof(mv_s->session_header),
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
char buf[1500];
struct mbuf *m;
struct uio *uio;
size_t len;

switch (mv_p->data_type) {
case MVXPSEC_DATA_MBUF:
m = mv_p->data_mbuf;
len = m->m_pkthdr.len;
if (len > sizeof(buf))
len = sizeof(buf);
m_copydata(m, 0, len, buf);
break;
case MVXPSEC_DATA_UIO:
uio = mv_p->data_uio;
len = uio->uio_resid;
if (len > sizeof(buf))
len = sizeof(buf);
cuio_copydata(uio, 0, len, buf);
break;
default:
len = 0;
}
if (len > 0)
mvxpsec_dump_data(__func__, buf, len);
}

if (mvxpsec_debug & MVXPSEC_DEBUG_PAYLOAD) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
"%s: session_descriptor:\n", __func__);
mvxpsec_dump_packet_desc(__func__, mv_p);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD,
"%s: session_data:\n", __func__);
mvxpsec_dump_packet_data(__func__, mv_p);
}

if (mvxpsec_debug & MVXPSEC_DEBUG_SRAM) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_SRAM,
"%s: SRAM\n", __func__);
mvxpsec_dump_sram(__func__, sc, 2000);
}

s = MVXPSEC_READ(sc, MV_ACC_STATUS);
if (s & MV_ACC_STATUS_MAC_ERR) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR,
"%s: Message Authentication Failed.\n", __func__);
}
}
#endif

/* copy back IV */
if (mv_p->flags & CRP_EXT_IV) {
memcpy(mv_p->ext_iv,
&mv_p->pkt_header.crp_iv_ext, mv_p->ext_ivlen);
mv_p->ext_iv = NULL;
mv_p->ext_ivlen = 0;
}

/* notify opencrypto */
mv_p->crp->crp_etype = 0;
crypto_done(mv_p->crp);
mv_p->crp = NULL;

/* unblock driver */
mvxpsec_packet_dealloc(mv_p);
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);

MVXPSEC_EVCNT_INCR(sc, packet_ok);

return 0;
}

/*
* Opencrypto API registration
*/
int
mvxpsec_register(struct mvxpsec_softc *sc)
{
int oplen = SRAM_PAYLOAD_SIZE;
int flags = 0;
int err;

sc->sc_nsessions = 0;
sc->sc_cid = crypto_get_driverid(0);
if (sc->sc_cid < 0) {
log(LOG_ERR,
"%s: crypto_get_driverid() failed.\n", __func__);
err = EINVAL;
goto done;
}

/* Ciphers */
err = crypto_register(sc->sc_cid, CRYPTO_DES_CBC, oplen, flags,
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
if (err)
goto done;

err = crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, oplen, flags,
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
if (err)
goto done;

err = crypto_register(sc->sc_cid, CRYPTO_AES_CBC, oplen, flags,
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
if (err)
goto done;

/* MACs */
err = crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96,
oplen, flags,
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
if (err)
goto done;

err = crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96,
oplen, flags,
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc);
if (err)
goto done;

#ifdef DEBUG
log(LOG_DEBUG,
"%s: registered to opencrypto(max data = %d bytes)\n",
device_xname(sc->sc_dev), oplen);
#endif

err = 0;
done:
return err;
}

/*
* Create new opencrypto session
*
* - register cipher key, mac key.
* - initialize mac internal state.
*/
int
mvxpsec_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_session *mv_s = NULL;
struct cryptoini *c;
static int hint = 0;
int session = -1;
int sid;
int err;
int i;

/* allocate driver session context */
mv_s = mvxpsec_session_alloc(sc);
if (mv_s == NULL)
return ENOMEM;

/*
* lookup opencrypto session table
*
* we have sc_session_mtx after here.
*/
mutex_enter(&sc->sc_session_mtx);
if (sc->sc_nsessions >= MVXPSEC_MAX_SESSIONS) {
mutex_exit(&sc->sc_session_mtx);
log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
__func__, MVXPSEC_MAX_SESSIONS);
mvxpsec_session_dealloc(mv_s);
return ENOMEM;
}
for (i = hint; i < MVXPSEC_MAX_SESSIONS; i++) {
if (sc->sc_sessions[i])
continue;
session = i;
hint = session + 1;
break;
}
if (session < 0) {
for (i = 0; i < hint; i++) {
if (sc->sc_sessions[i])
continue;
session = i;
hint = session + 1;
break;
}
if (session < 0) {
mutex_exit(&sc->sc_session_mtx);
/* session full */
log(LOG_ERR, "%s: too many IPsec SA(max %d)\n",
__func__, MVXPSEC_MAX_SESSIONS);
mvxpsec_session_dealloc(mv_s);
hint = 0;
return ENOMEM;
}
}
if (hint >= MVXPSEC_MAX_SESSIONS)
hint = 0;
sc->sc_nsessions++;
sc->sc_sessions[session] = mv_s;
#ifdef DEBUG
log(LOG_DEBUG, "%s: new session %d allocated\n", __func__, session);
#endif

sid = MVXPSEC_SID(device_unit(sc->sc_dev), session);
mv_s->sid = sid;

/* setup the session key ... */
for (c = cri; c; c = c->cri_next) {
switch (c->cri_alg) {
case CRYPTO_DES_CBC:
case CRYPTO_3DES_CBC:
case CRYPTO_AES_CBC:
/* key */
if (mvxpsec_key_precomp(c->cri_alg,
c->cri_key, c->cri_klen,
&mv_s->session_header.crp_key,
&mv_s->session_header.crp_key_d)) {
log(LOG_ERR,
"%s: Invalid HMAC key for %s.\n",
__func__, s_ctlalg(c->cri_alg));
err = EINVAL;
goto fail;
}
if (mv_s->sflags & RDY_CRP_KEY) {
log(LOG_WARNING,
"%s: overwrite cipher: %s->%s.\n",
__func__,
s_ctlalg(mv_s->cipher_alg),
s_ctlalg(c->cri_alg));
}
mv_s->sflags |= RDY_CRP_KEY;
mv_s->enc_klen = c->cri_klen;
mv_s->cipher_alg = c->cri_alg;
/* create per session IV (compatible with KAME IPsec) */
cprng_fast(&mv_s->session_iv, sizeof(mv_s->session_iv));
mv_s->sflags |= RDY_CRP_IV;
break;
case CRYPTO_SHA1_HMAC_96:
case CRYPTO_MD5_HMAC_96:
/* key */
if (mvxpsec_hmac_precomp(c->cri_alg,
c->cri_key, c->cri_klen,
(uint32_t *)&mv_s->session_header.miv_in,
(uint32_t *)&mv_s->session_header.miv_out)) {
log(LOG_ERR,
"%s: Invalid MAC key\n", __func__);
err = EINVAL;
goto fail;
}
if (mv_s->sflags & RDY_MAC_KEY ||
mv_s->sflags & RDY_MAC_IV) {
log(LOG_ERR,
"%s: overwrite HMAC: %s->%s.\n",
__func__, s_ctlalg(mv_s->hmac_alg),
s_ctlalg(c->cri_alg));
}
mv_s->sflags |= RDY_MAC_KEY;
mv_s->sflags |= RDY_MAC_IV;

mv_s->mac_klen = c->cri_klen;
mv_s->hmac_alg = c->cri_alg;
break;
default:
log(LOG_ERR, "%s: Unknown algorithm %d\n",
__func__, c->cri_alg);
err = EINVAL;
goto fail;
}
}
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"H/W Crypto session (id:%u) added.\n", session);

*sidp = sid;
MVXPSEC_EVCNT_INCR(sc, session_new);
mutex_exit(&sc->sc_session_mtx);

/* sync session header(it's never touched after here) */
bus_dmamap_sync(sc->sc_dmat,
mv_s->session_header_map,
0, sizeof(mv_s->session_header),
BUS_DMASYNC_PREWRITE);

return 0;

fail:
sc->sc_nsessions--;
sc->sc_sessions[session] = NULL;
hint = session;
if (mv_s)
mvxpsec_session_dealloc(mv_s);
log(LOG_WARNING,
"%s: Failed to add H/W crypto session (id:%u): err=%d\n",
__func__, session, err);

mutex_exit(&sc->sc_session_mtx);
return err;
}

/*
* remove opencrypto session
*/
void
mvxpsec_freesession(void *arg, uint64_t tid)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_session *mv_s;
int session;
uint32_t sid = ((uint32_t)tid) & 0xffffffff;

session = MVXPSEC_SESSION(sid);
KASSERTMSG(session >= 0, "session=%d", session);
KASSERTMSG(session < MVXPSEC_MAX_SESSIONS, "session=%d max=%d",
session, MVXPSEC_MAX_SESSIONS);

mutex_enter(&sc->sc_session_mtx);
mv_s = sc->sc_sessions[session];
KASSERT(mv_s != NULL);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: inactivate session %d\n", __func__, session);

/* inactivate mvxpsec session */
sc->sc_sessions[session] = NULL;
sc->sc_nsessions--;
sc->sc_last_session = NULL;
mutex_exit(&sc->sc_session_mtx);

KASSERT(sc->sc_nsessions >= 0);
KASSERT(mv_s->sid == sid);

mvxpsec_session_dealloc(mv_s);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"H/W Crypto session (id: %d) deleted.\n", session);

/* force unblock opencrypto */
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);

MVXPSEC_EVCNT_INCR(sc, session_free);
}

/*
* process data with existing session
*/
int
mvxpsec_dispatch(void *arg, struct cryptop *crp, int hint)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_session *mv_s;
struct mvxpsec_packet *mv_p;
int q_full;
int running;
int err;

mutex_enter(&sc->sc_queue_mtx);

/*
* lookup session
*/
mutex_enter(&sc->sc_session_mtx);
mv_s = mvxpsec_session_lookup(sc, crp->crp_sid);
if (__predict_false(mv_s == NULL)) {
err = EINVAL;
mv_p = NULL;
mutex_exit(&sc->sc_session_mtx);
goto fail;
}
mv_p = mvxpsec_packet_alloc(mv_s);
if (__predict_false(mv_p == NULL)) {
mutex_exit(&sc->sc_session_mtx);
mutex_exit(&sc->sc_queue_mtx);
return ERESTART; /* => queued in opencrypto layer */
}
mutex_exit(&sc->sc_session_mtx);

/*
* check queue status
*/
#ifdef MVXPSEC_MULTI_PACKET
q_full = (sc->sc_wait_qlen >= sc->sc_wait_qlimit) ? 1 : 0;
#else
q_full = (sc->sc_wait_qlen != 0) ? 1 : 0;
#endif
running = (sc->sc_flags & HW_RUNNING) ? 1: 0;
if (q_full) {
/* input queue is full. */
if (!running && sc->sc_wait_qlen > 0)
mvxpsec_dispatch_queue(sc);
MVXPSEC_EVCNT_INCR(sc, queue_full);
mvxpsec_packet_dealloc(mv_p);
mutex_exit(&sc->sc_queue_mtx);
return ERESTART; /* => queued in opencrypto layer */
}

/*
* Load and setup packet data
*/
err = mvxpsec_packet_setcrp(mv_p, crp);
if (__predict_false(err))
goto fail;

/*
* Setup DMA descriptor chains
*/
mutex_enter(&sc->sc_dma_mtx);
err = mvxpsec_dma_copy_packet(sc, mv_p);
mutex_exit(&sc->sc_dma_mtx);
if (__predict_false(err))
goto fail;

#ifdef MVXPSEC_DEBUG
mvxpsec_dump_packet(__func__, mv_p);
#endif

/*
* Sync/inval the data cache
*/
err = mvxpsec_dma_sync_packet(sc, mv_p);
if (__predict_false(err))
goto fail;

/*
* Enqueue the packet
*/
MVXPSEC_EVCNT_INCR(sc, dispatch_packets);
#ifdef MVXPSEC_MULTI_PACKET
mvxpsec_packet_enqueue(mv_p);
if (!running)
mvxpsec_dispatch_queue(sc);
#else
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue);
sc->sc_wait_qlen++;
mv_p->flags |= SETUP_DONE;
if (!running)
mvxpsec_dispatch_queue(sc);
#endif
mutex_exit(&sc->sc_queue_mtx);
return 0;

fail:
/* Drop the incoming packet */
mvxpsec_drop(sc, crp, mv_p, err);
mutex_exit(&sc->sc_queue_mtx);
return 0;
}

/*
* back the packet to the IP stack
*/
void
mvxpsec_done(void *arg)
{
struct mvxpsec_softc *sc = arg;
struct mvxpsec_packet *mv_p;
mvxpsec_queue_t ret_queue;
int ndone;

mutex_enter(&sc->sc_queue_mtx);

/* stop wdog timer */
callout_stop(&sc->sc_timeout);

/* refill MVXPSEC */
ret_queue = sc->sc_run_queue;
SIMPLEQ_INIT(&sc->sc_run_queue);
sc->sc_flags &= ~HW_RUNNING;
if (sc->sc_wait_qlen > 0)
mvxpsec_dispatch_queue(sc);

ndone = 0;
while ( (mv_p = SIMPLEQ_FIRST(&ret_queue)) != NULL) {
SIMPLEQ_REMOVE_HEAD(&ret_queue, queue);
mvxpsec_dma_free(sc, &mv_p->dma_ring);
mvxpsec_done_packet(mv_p);
ndone++;
}
MVXPSEC_EVCNT_MAX(sc, max_done, ndone);

mutex_exit(&sc->sc_queue_mtx);
}

/*
* drop the packet
*/
INLINE void
mvxpsec_drop(struct mvxpsec_softc *sc, struct cryptop *crp,
struct mvxpsec_packet *mv_p, int err)
{
/* must called with sc->sc_queue_mtx held */
KASSERT(mutex_owned(&sc->sc_queue_mtx));

if (mv_p)
mvxpsec_packet_dealloc(mv_p);
if (err < 0)
err = EINVAL;
crp->crp_etype = err;
crypto_done(crp);
MVXPSEC_EVCNT_INCR(sc, packet_err);

/* dispatch other packets in queue */
if (sc->sc_wait_qlen > 0 &&
!(sc->sc_flags & HW_RUNNING))
mvxpsec_dispatch_queue(sc);

/* unblock driver for dropped packet */
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ);
}

/* move wait queue entry to run queue */
STATIC int
mvxpsec_dispatch_queue(struct mvxpsec_softc *sc)
{
struct mvxpsec_packet *mv_p;
paddr_t head;
int ndispatch = 0;

/* must called with sc->sc_queue_mtx held */
KASSERT(mutex_owned(&sc->sc_queue_mtx));

/* check there is any task */
if (__predict_false(sc->sc_flags & HW_RUNNING)) {
log(LOG_WARNING,
"%s: another packet already exist.\n", __func__);
return 0;
}
if (__predict_false(SIMPLEQ_EMPTY(&sc->sc_wait_queue))) {
log(LOG_WARNING,
"%s: no waiting packet yet(qlen=%d).\n",
__func__, sc->sc_wait_qlen);
return 0;
}

/* move queue */
sc->sc_run_queue = sc->sc_wait_queue;
sc->sc_flags |= HW_RUNNING; /* dropped by intr or timeout */
SIMPLEQ_INIT(&sc->sc_wait_queue);
ndispatch = sc->sc_wait_qlen;
sc->sc_wait_qlen = 0;

/* get 1st DMA descriptor */
mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue);
head = mv_p->dma_ring.dma_head->phys_addr;

/* terminate last DMA descriptor */
mv_p = SIMPLEQ_LAST(&sc->sc_run_queue, mvxpsec_packet, queue);
mvxpsec_dma_finalize(sc, &mv_p->dma_ring);

/* configure TDMA */
if (mvxpsec_dma_wait(sc) < 0) {
log(LOG_ERR, "%s: DMA DEVICE not responding", __func__);
callout_schedule(&sc->sc_timeout, hz);
return 0;
}
MVXPSEC_WRITE(sc, MV_TDMA_NXT, head);

/* trigger ACC */
if (mvxpsec_acc_wait(sc) < 0) {
log(LOG_ERR, "%s: MVXPSEC not responding", __func__);
callout_schedule(&sc->sc_timeout, hz);
return 0;
}
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_ACT);

MVXPSEC_EVCNT_MAX(sc, max_dispatch, ndispatch);
MVXPSEC_EVCNT_INCR(sc, dispatch_queue);
callout_schedule(&sc->sc_timeout, hz);
return 0;
}

/*
* process opencrypto operations(cryptop) for packets.
*/
INLINE int
mvxpsec_parse_crd(struct mvxpsec_packet *mv_p, struct cryptodesc *crd)
{
int ivlen;

KASSERT(mv_p->flags & RDY_DATA);

/* MAC & Ciphers: set data location and operation */
switch (crd->crd_alg) {
case CRYPTO_SHA1_HMAC_96:
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
/* fall through */
case CRYPTO_SHA1_HMAC:
mv_p->mac_dst = crd->crd_inject;
mv_p->mac_off = crd->crd_skip;
mv_p->mac_len = crd->crd_len;
MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
MV_ACC_CRYPTO_MAC_HMAC_SHA1);
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
/* No more setup for MAC */
return 0;
case CRYPTO_MD5_HMAC_96:
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96;
/* fall through */
case CRYPTO_MD5_HMAC:
mv_p->mac_dst = crd->crd_inject;
mv_p->mac_off = crd->crd_skip;
mv_p->mac_len = crd->crd_len;
MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config,
MV_ACC_CRYPTO_MAC_HMAC_MD5);
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC);
/* No more setup for MAC */
return 0;
case CRYPTO_DES_CBC:
mv_p->enc_ivoff = crd->crd_inject;
mv_p->enc_off = crd->crd_skip;
mv_p->enc_len = crd->crd_len;
ivlen = 8;
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
MV_ACC_CRYPTO_ENC_DES);
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
break;
case CRYPTO_3DES_CBC:
mv_p->enc_ivoff = crd->crd_inject;
mv_p->enc_off = crd->crd_skip;
mv_p->enc_len = crd->crd_len;
ivlen = 8;
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
MV_ACC_CRYPTO_ENC_3DES);
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_3DES_EDE;
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
break;
case CRYPTO_AES_CBC:
mv_p->enc_ivoff = crd->crd_inject;
mv_p->enc_off = crd->crd_skip;
mv_p->enc_len = crd->crd_len;
ivlen = 16;
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config,
MV_ACC_CRYPTO_ENC_AES);
MV_ACC_CRYPTO_AES_KLEN_SET(
mv_p->pkt_header.desc.acc_config,
mvxpsec_aesklen(mv_p->mv_s->enc_klen));
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC;
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC);
break;
default:
log(LOG_ERR, "%s: Unknown algorithm %d\n",
__func__, crd->crd_alg);
return EINVAL;
}

/* Operations only for Cipher, not MAC */
if (crd->crd_flags & CRD_F_ENCRYPT) {
/* Ciphers: Originate IV for Encryption.*/
mv_p->pkt_header.desc.acc_config &= ~MV_ACC_CRYPTO_DECRYPT;
mv_p->flags |= DIR_ENCRYPT;

if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "EXPLICIT IV\n");
mv_p->flags |= CRP_EXT_IV;
mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
}
else if (crd->crd_flags & CRD_F_IV_PRESENT) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "IV is present\n");
mvxpsec_packet_copy_iv(mv_p, crd->crd_inject, ivlen);
}
else {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "Create New IV\n");
mvxpsec_packet_write_iv(mv_p, NULL, ivlen);
}
}
else {
/* Ciphers: IV is loadded from crd_inject when it's present */
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_DECRYPT;
mv_p->flags |= DIR_DECRYPT;

if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
#ifdef MVXPSEC_DEBUG
if (mvxpsec_debug & MVXPSEC_DEBUG_ENC_IV) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV,
"EXPLICIT IV(Decrypt)\n");
mvxpsec_dump_data(__func__, crd->crd_iv, ivlen);
}
#endif
mv_p->flags |= CRP_EXT_IV;
mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen);
mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF;
}
}

KASSERT(!((mv_p->flags & DIR_ENCRYPT) && (mv_p->flags & DIR_DECRYPT)));

return 0;
}

INLINE int
mvxpsec_parse_crp(struct mvxpsec_packet *mv_p)
{
struct cryptop *crp = mv_p->crp;
struct cryptodesc *crd;
int err;

KASSERT(crp);

mvxpsec_packet_reset_op(mv_p);

for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
err = mvxpsec_parse_crd(mv_p, crd);
if (err)
return err;
}

return 0;
}

INLINE int
mvxpsec_packet_setcrp(struct mvxpsec_packet *mv_p, struct cryptop *crp)
{
int err = EINVAL;

/* register crp to the MVXPSEC packet */
if (crp->crp_flags & CRYPTO_F_IMBUF) {
err = mvxpsec_packet_setmbuf(mv_p,
(struct mbuf *)crp->crp_buf);
mv_p->crp = crp;
}
else if (crp->crp_flags & CRYPTO_F_IOV) {
err = mvxpsec_packet_setuio(mv_p,
(struct uio *)crp->crp_buf);
mv_p->crp = crp;
}
else {
err = mvxpsec_packet_setdata(mv_p,
(struct mbuf *)crp->crp_buf, crp->crp_ilen);
mv_p->crp = crp;
}
if (__predict_false(err))
return err;

/* parse crp and setup MVXPSEC registers/descriptors */
err = mvxpsec_parse_crp(mv_p);
if (__predict_false(err))
return err;

/* fixup data offset to fit MVXPSEC internal SRAM */
err = mvxpsec_header_finalize(mv_p);
if (__predict_false(err))
return err;

return 0;
}

/*
* load data for encrypt/decrypt/authentication
*
* data is raw kernel memory area.
*/
STATIC int
mvxpsec_packet_setdata(struct mvxpsec_packet *mv_p,
void *data, uint32_t data_len)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
struct mvxpsec_softc *sc = mv_s->sc;

if (bus_dmamap_load(sc->sc_dmat, mv_p->data_map, data, data_len,
NULL, BUS_DMA_NOWAIT)) {
log(LOG_ERR, "%s: cannot load data\n", __func__);
return -1;
}
mv_p->data_type = MVXPSEC_DATA_RAW;
mv_p->data_raw = data;
mv_p->data_len = data_len;
mv_p->flags |= RDY_DATA;

return 0;
}

/*
* load data for encrypt/decrypt/authentication
*
* data is mbuf based network data.
*/
STATIC int
mvxpsec_packet_setmbuf(struct mvxpsec_packet *mv_p, struct mbuf *m)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
struct mvxpsec_softc *sc = mv_s->sc;
size_t pktlen = 0;

if (__predict_true(m->m_flags & M_PKTHDR))
pktlen = m->m_pkthdr.len;
else {
struct mbuf *mp = m;

while (mp != NULL) {
pktlen += m->m_len;
mp = mp->m_next;
}
}
if (pktlen > SRAM_PAYLOAD_SIZE) {
#if NIPSEC > 0
extern percpu_t *espstat_percpu;
/* XXX:
* layer violation. opencrypto knows our max packet size
* from crypto_register(9) API.
*/

_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
#endif
log(LOG_ERR,
"%s: ESP Packet too large: %zu [oct.] > %zu [oct.]\n",
device_xname(sc->sc_dev),
(size_t)pktlen, SRAM_PAYLOAD_SIZE);
mv_p->data_type = MVXPSEC_DATA_NONE;
mv_p->data_mbuf = NULL;
return -1;
}

if (bus_dmamap_load_mbuf(sc->sc_dmat, mv_p->data_map, m,
BUS_DMA_NOWAIT)) {
mv_p->data_type = MVXPSEC_DATA_NONE;
mv_p->data_mbuf = NULL;
log(LOG_ERR, "%s: cannot load mbuf\n", __func__);
return -1;
}

/* set payload buffer */
mv_p->data_type = MVXPSEC_DATA_MBUF;
mv_p->data_mbuf = m;
if (m->m_flags & M_PKTHDR) {
mv_p->data_len = m->m_pkthdr.len;
}
else {
mv_p->data_len = 0;
while (m) {
mv_p->data_len += m->m_len;
m = m->m_next;
}
}
mv_p->flags |= RDY_DATA;

return 0;
}

STATIC int
mvxpsec_packet_setuio(struct mvxpsec_packet *mv_p, struct uio *uio)
{
struct mvxpsec_session *mv_s = mv_p->mv_s;
struct mvxpsec_softc *sc = mv_s->sc;

if (uio->uio_resid > SRAM_PAYLOAD_SIZE) {
#if NIPSEC > 0
extern percpu_t *espstat_percpu;
/* XXX:
* layer violation. opencrypto knows our max packet size
* from crypto_register(9) API.
*/

_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG);
#endif
log(LOG_ERR,
"%s: uio request too large: %zu [oct.] > %zu [oct.]\n",
device_xname(sc->sc_dev),
uio->uio_resid, SRAM_PAYLOAD_SIZE);
mv_p->data_type = MVXPSEC_DATA_NONE;
mv_p->data_mbuf = NULL;
return -1;
}

if (bus_dmamap_load_uio(sc->sc_dmat, mv_p->data_map, uio,
BUS_DMA_NOWAIT)) {
mv_p->data_type = MVXPSEC_DATA_NONE;
mv_p->data_mbuf = NULL;
log(LOG_ERR, "%s: cannot load uio buf\n", __func__);
return -1;
}

/* set payload buffer */
mv_p->data_type = MVXPSEC_DATA_UIO;
mv_p->data_uio = uio;
mv_p->data_len = uio->uio_resid;
mv_p->flags |= RDY_DATA;

return 0;
}

STATIC int
mvxpsec_packet_rdata(struct mvxpsec_packet *mv_p,
int off, int len, void *cp)
{
uint8_t *p;

if (mv_p->data_type == MVXPSEC_DATA_RAW) {
p = (uint8_t *)mv_p->data_raw + off;
memcpy(cp, p, len);
}
else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
m_copydata(mv_p->data_mbuf, off, len, cp);
}
else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
cuio_copydata(mv_p->data_uio, off, len, cp);
}
else
return -1;

return 0;
}

STATIC int
mvxpsec_packet_wdata(struct mvxpsec_packet *mv_p,
int off, int len, void *cp)
{
uint8_t *p;

if (mv_p->data_type == MVXPSEC_DATA_RAW) {
p = (uint8_t *)mv_p->data_raw + off;
memcpy(p, cp, len);
}
else if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
m_copyback(mv_p->data_mbuf, off, len, cp);
}
else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
cuio_copyback(mv_p->data_uio, off, len, cp);
}
else
return -1;

return 0;
}

/*
* Set initial vector of cipher to the session.
*/
STATIC int
mvxpsec_packet_write_iv(struct mvxpsec_packet *mv_p, void *iv, int ivlen)
{
uint8_t ivbuf[16];

KASSERT(ivlen == 8 || ivlen == 16);

if (iv == NULL) {
if (mv_p->mv_s->sflags & RDY_CRP_IV) {
/* use per session IV (compatible with KAME IPsec) */
mv_p->pkt_header.crp_iv_work = mv_p->mv_s->session_iv;
mv_p->flags |= RDY_CRP_IV;
return 0;
}
cprng_fast(ivbuf, ivlen);
iv = ivbuf;
}
memcpy(&mv_p->pkt_header.crp_iv_work, iv, ivlen);
if (mv_p->flags & CRP_EXT_IV) {
memcpy(&mv_p->pkt_header.crp_iv_ext, iv, ivlen);
mv_p->ext_iv = iv;
mv_p->ext_ivlen = ivlen;
}
mv_p->flags |= RDY_CRP_IV;

return 0;
}

STATIC int
mvxpsec_packet_copy_iv(struct mvxpsec_packet *mv_p, int off, int ivlen)
{
mvxpsec_packet_rdata(mv_p, off, ivlen,
&mv_p->pkt_header.crp_iv_work);
mv_p->flags |= RDY_CRP_IV;

return 0;
}

/*
* set a encryption or decryption key to the session
*
* Input key material is big endian.
*/
STATIC int
mvxpsec_key_precomp(int alg, void *keymat, int kbitlen,
void *key_encrypt, void *key_decrypt)
{
uint32_t *kp = keymat;
uint32_t *ekp = key_encrypt;
uint32_t *dkp = key_decrypt;
int i;

switch (alg) {
case CRYPTO_DES_CBC:
if (kbitlen < 64 || (kbitlen % 8) != 0) {
log(LOG_WARNING,
"mvxpsec: invalid DES keylen %d\n", kbitlen);
return EINVAL;
}
for (i = 0; i < 2; i++)
dkp[i] = ekp[i] = kp[i];
for (; i < 8; i++)
dkp[i] = ekp[i] = 0;
break;
case CRYPTO_3DES_CBC:
if (kbitlen < 192 || (kbitlen % 8) != 0) {
log(LOG_WARNING,
"mvxpsec: invalid 3DES keylen %d\n", kbitlen);
return EINVAL;
}
for (i = 0; i < 8; i++)
dkp[i] = ekp[i] = kp[i];
break;
case CRYPTO_AES_CBC:
if (kbitlen < 128) {
log(LOG_WARNING,
"mvxpsec: invalid AES keylen %d\n", kbitlen);
return EINVAL;
}
else if (kbitlen < 192) {
/* AES-128 */
for (i = 0; i < 4; i++)
ekp[i] = kp[i];
for (; i < 8; i++)
ekp[i] = 0;
}
else if (kbitlen < 256) {
/* AES-192 */
for (i = 0; i < 6; i++)
ekp[i] = kp[i];
for (; i < 8; i++)
ekp[i] = 0;
}
else {
/* AES-256 */
for (i = 0; i < 8; i++)
ekp[i] = kp[i];
}
/* make decryption key */
mv_aes_deckey((uint8_t *)dkp, (uint8_t *)ekp, kbitlen);
break;
default:
for (i = 0; i < 8; i++)
ekp[0] = dkp[0] = 0;
break;
}

#ifdef MVXPSEC_DEBUG
if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: keyregistered\n", __func__);
mvxpsec_dump_data(__func__, ekp, 32);
}
#endif

return 0;
}

/*
* set MAC key to the session
*
* MAC engine has no register for key itself, but the engine has
* inner and outer IV register. software must compute IV before
* enable the engine.
*
* IV is a hash of ipad/opad. these are defined by FIPS-198a
* standard.
*/
STATIC int
mvxpsec_hmac_precomp(int alg, void *key, int kbitlen,
void *iv_inner, void *iv_outer)
{
SHA1_CTX sha1;
MD5_CTX md5;
uint8_t *key8 = key;
uint8_t kbuf[64];
uint8_t ipad[64];
uint8_t opad[64];
uint32_t *iv_in = iv_inner;
uint32_t *iv_out = iv_outer;
int kbytelen;
int i;
#define HMAC_IPAD 0x36
#define HMAC_OPAD 0x5c

kbytelen = kbitlen / 8;
KASSERT(kbitlen == kbytelen * 8);
if (kbytelen > 64) {
SHA1Init(&sha1);
SHA1Update(&sha1, key, kbytelen);
SHA1Final(kbuf, &sha1);
key8 = kbuf;
kbytelen = 64;
}

/* make initial 64 oct. string */
switch (alg) {
case CRYPTO_SHA1_HMAC_96:
case CRYPTO_SHA1_HMAC:
case CRYPTO_MD5_HMAC_96:
case CRYPTO_MD5_HMAC:
for (i = 0; i < kbytelen; i++) {
ipad[i] = (key8[i] ^ HMAC_IPAD);
opad[i] = (key8[i] ^ HMAC_OPAD);
}
for (; i < 64; i++) {
ipad[i] = HMAC_IPAD;
opad[i] = HMAC_OPAD;
}
break;
default:
break;
}
#ifdef MVXPSEC_DEBUG
if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: HMAC-KEY Pre-comp:\n", __func__);
mvxpsec_dump_data(__func__, key, 64);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: ipad:\n", __func__);
mvxpsec_dump_data(__func__, ipad, sizeof(ipad));
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: opad:\n", __func__);
mvxpsec_dump_data(__func__, opad, sizeof(opad));
}
#endif

/* make iv from string */
switch (alg) {
case CRYPTO_SHA1_HMAC_96:
case CRYPTO_SHA1_HMAC:
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: Generate iv_in(SHA1)\n", __func__);
SHA1Init(&sha1);
SHA1Update(&sha1, ipad, 64);
/* XXX: private state... (LE) */
iv_in[0] = htobe32(sha1.state[0]);
iv_in[1] = htobe32(sha1.state[1]);
iv_in[2] = htobe32(sha1.state[2]);
iv_in[3] = htobe32(sha1.state[3]);
iv_in[4] = htobe32(sha1.state[4]);

MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: Generate iv_out(SHA1)\n", __func__);
SHA1Init(&sha1);
SHA1Update(&sha1, opad, 64);
/* XXX: private state... (LE) */
iv_out[0] = htobe32(sha1.state[0]);
iv_out[1] = htobe32(sha1.state[1]);
iv_out[2] = htobe32(sha1.state[2]);
iv_out[3] = htobe32(sha1.state[3]);
iv_out[4] = htobe32(sha1.state[4]);
break;
case CRYPTO_MD5_HMAC_96:
case CRYPTO_MD5_HMAC:
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: Generate iv_in(MD5)\n", __func__);
MD5Init(&md5);
MD5Update(&md5, ipad, sizeof(ipad));
/* XXX: private state... (LE) */
iv_in[0] = htobe32(md5.state[0]);
iv_in[1] = htobe32(md5.state[1]);
iv_in[2] = htobe32(md5.state[2]);
iv_in[3] = htobe32(md5.state[3]);
iv_in[4] = 0;

MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO,
"%s: Generate iv_out(MD5)\n", __func__);
MD5Init(&md5);
MD5Update(&md5, opad, sizeof(opad));
/* XXX: private state... (LE) */
iv_out[0] = htobe32(md5.state[0]);
iv_out[1] = htobe32(md5.state[1]);
iv_out[2] = htobe32(md5.state[2]);
iv_out[3] = htobe32(md5.state[3]);
iv_out[4] = 0;
break;
default:
break;
}

#ifdef MVXPSEC_DEBUG
if (mvxpsec_debug & MVXPSEC_DEBUG_HASH_IV) {
MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
"%s: HMAC IV-IN\n", __func__);
mvxpsec_dump_data(__func__, (uint8_t *)iv_in, 20);
MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV,
"%s: HMAC IV-OUT\n", __func__);
mvxpsec_dump_data(__func__, (uint8_t *)iv_out, 20);
}
#endif

return 0;
#undef HMAC_IPAD
#undef HMAC_OPAD
}

/*
* AES Support routine
*/
static uint8_t AES_SBOX[256] = {
99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215,
171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175,
156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165,
229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154,
7, 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110,
90, 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237,
32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239,
170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255,
243, 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61,
100, 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238,
184, 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92,
194, 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213,
78, 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46,
28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62,
181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85,
40, 223, 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15,
176, 84, 187, 22
};

static uint32_t AES_RCON[30] = {
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8,
0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4,
0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91
};

STATIC int
mv_aes_ksched(uint8_t k[4][MAXKC], int keyBits,
uint8_t W[MAXROUNDS+1][4][MAXBC])
{
int KC, BC, ROUNDS;
int i, j, t, rconpointer = 0;
uint8_t tk[4][MAXKC];

switch (keyBits) {
case 128:
ROUNDS = 10;
KC = 4;
break;
case 192:
ROUNDS = 12;
KC = 6;
break;
case 256:
ROUNDS = 14;
KC = 8;
break;
default:
return (-1);
}
BC = 4; /* 128 bits */

for(j = 0; j < KC; j++)
for(i = 0; i < 4; i++)
tk[i][j] = k[i][j];
t = 0;

/* copy values into round key array */
for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];

while (t < (ROUNDS+1)*BC) { /* while not enough round key material calculated */
/* calculate new values */
for(i = 0; i < 4; i++)
tk[i][0] ^= AES_SBOX[tk[(i+1)%4][KC-1]];
tk[0][0] ^= AES_RCON[rconpointer++];

if (KC != 8)
for(j = 1; j < KC; j++)
for(i = 0; i < 4; i++)
tk[i][j] ^= tk[i][j-1];
else {
for(j = 1; j < KC/2; j++)
for(i = 0; i < 4; i++)
tk[i][j] ^= tk[i][j-1];
for(i = 0; i < 4; i++)
tk[i][KC/2] ^= AES_SBOX[tk[i][KC/2 - 1]];
for(j = KC/2 + 1; j < KC; j++)
for(i = 0; i < 4; i++)
tk[i][j] ^= tk[i][j-1];
}
/* copy values into round key array */
for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++)
for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j];
}

return 0;
}

STATIC int
mv_aes_deckey(uint8_t *expandedKey, uint8_t *keyMaterial, int keyLen)
{
uint8_t W[MAXROUNDS+1][4][MAXBC];
uint8_t k[4][MAXKC];
uint8_t j;
int i, rounds, KC;

if (expandedKey == NULL)
return -1;

if (!((keyLen == 128) || (keyLen == 192) || (keyLen == 256)))
return -1;

if (keyMaterial == NULL)
return -1;

/* initialize key schedule: */
for (i=0; i<keyLen/8; i++) {
j = keyMaterial[i];
k[i % 4][i / 4] = j;
}

mv_aes_ksched(k, keyLen, W);
switch (keyLen) {
case 128:
rounds = 10;
KC = 4;
break;
case 192:
rounds = 12;
KC = 6;
break;
case 256:
rounds = 14;
KC = 8;
break;
default:
return -1;
}

for(i=0; i<MAXBC; i++)
for(j=0; j<4; j++)
expandedKey[i*4+j] = W[rounds][j][i];
for(; i<KC; i++)
for(j=0; j<4; j++)
expandedKey[i*4+j] = W[rounds-1][j][i+MAXBC-KC];

return 0;
}

/*
* Clear cipher/mac operation state
*/
INLINE void
mvxpsec_packet_reset_op(struct mvxpsec_packet *mv_p)
{
mv_p->pkt_header.desc.acc_config = 0;
mv_p->enc_off = mv_p->enc_ivoff = mv_p->enc_len = 0;
mv_p->mac_off = mv_p->mac_dst = mv_p->mac_len = 0;
}

/*
* update MVXPSEC operation order
*/
INLINE void
mvxpsec_packet_update_op_order(struct mvxpsec_packet *mv_p, int op)
{
struct mvxpsec_acc_descriptor *acc_desc = &mv_p->pkt_header.desc;
uint32_t cur_op = acc_desc->acc_config & MV_ACC_CRYPTO_OP_MASK;

KASSERT(op == MV_ACC_CRYPTO_OP_MAC || op == MV_ACC_CRYPTO_OP_ENC);
KASSERT((op & MV_ACC_CRYPTO_OP_MASK) == op);

if (cur_op == 0)
acc_desc->acc_config |= op;
else if (cur_op == MV_ACC_CRYPTO_OP_MAC && op == MV_ACC_CRYPTO_OP_ENC) {
acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
acc_desc->acc_config |= MV_ACC_CRYPTO_OP_MACENC;
/* MAC then ENC (= decryption) */
}
else if (cur_op == MV_ACC_CRYPTO_OP_ENC && op == MV_ACC_CRYPTO_OP_MAC) {
acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK;
acc_desc->acc_config |= MV_ACC_CRYPTO_OP_ENCMAC;
/* ENC then MAC (= encryption) */
}
else {
log(LOG_ERR, "%s: multiple %s algorithm is not supported.\n",
__func__,
(op == MV_ACC_CRYPTO_OP_ENC) ? "encryption" : "authentication");
}
}

/*
* Parameter Conversions
*/
INLINE uint32_t
mvxpsec_alg2acc(uint32_t alg)
{
uint32_t reg;

switch (alg) {
case CRYPTO_DES_CBC:
reg = MV_ACC_CRYPTO_ENC_DES;
reg |= MV_ACC_CRYPTO_CBC;
break;
case CRYPTO_3DES_CBC:
reg = MV_ACC_CRYPTO_ENC_3DES;
reg |= MV_ACC_CRYPTO_3DES_EDE;
reg |= MV_ACC_CRYPTO_CBC;
break;
case CRYPTO_AES_CBC:
reg = MV_ACC_CRYPTO_ENC_AES;
reg |= MV_ACC_CRYPTO_CBC;
break;
case CRYPTO_SHA1_HMAC_96:
reg = MV_ACC_CRYPTO_MAC_HMAC_SHA1;
reg |= MV_ACC_CRYPTO_MAC_96;
break;
case CRYPTO_MD5_HMAC_96:
reg = MV_ACC_CRYPTO_MAC_HMAC_MD5;
reg |= MV_ACC_CRYPTO_MAC_96;
break;
default:
reg = 0;
break;
}

return reg;
}

INLINE uint32_t
mvxpsec_aesklen(int klen)
{
if (klen < 128)
return 0;
else if (klen < 192)
return MV_ACC_CRYPTO_AES_KLEN_128;
else if (klen < 256)
return MV_ACC_CRYPTO_AES_KLEN_192;
else
return MV_ACC_CRYPTO_AES_KLEN_256;

return 0;
}

/*
* String Conversions
*/
STATIC const char *
s_errreg(uint32_t v)
{
static char buf[80];

snprintf(buf, sizeof(buf),
"%sMiss %sDoubleHit %sBothHit %sDataError",
(v & MV_TDMA_ERRC_MISS) ? "+" : "-",
(v & MV_TDMA_ERRC_DHIT) ? "+" : "-",
(v & MV_TDMA_ERRC_BHIT) ? "+" : "-",
(v & MV_TDMA_ERRC_DERR) ? "+" : "-");

return (const char *)buf;
}

STATIC const char *
s_winreg(uint32_t v)
{
static char buf[80];

snprintf(buf, sizeof(buf),
"%s TGT 0x%x ATTR 0x%02x size %u(0x%04x)[64KB]",
(v & MV_TDMA_ATTR_ENABLE) ? "EN" : "DIS",
MV_TDMA_ATTR_GET_TARGET(v), MV_TDMA_ATTR_GET_ATTR(v),
MV_TDMA_ATTR_GET_SIZE(v), MV_TDMA_ATTR_GET_SIZE(v));

return (const char *)buf;
}

STATIC const char *
s_ctrlreg(uint32_t reg)
{
static char buf[80];

snprintf(buf, sizeof(buf),
"%s: %sFETCH DBURST-%u SBURST-%u %sOUTS %sCHAIN %sBSWAP %sACT",
(reg & MV_TDMA_CONTROL_ENABLE) ? "ENABLE" : "DISABLE",
(reg & MV_TDMA_CONTROL_FETCH) ? "+" : "-",
MV_TDMA_CONTROL_GET_DST_BURST(reg),
MV_TDMA_CONTROL_GET_SRC_BURST(reg),
(reg & MV_TDMA_CONTROL_OUTS_EN) ? "+" : "-",
(reg & MV_TDMA_CONTROL_CHAIN_DIS) ? "-" : "+",
(reg & MV_TDMA_CONTROL_BSWAP_DIS) ? "-" : "+",
(reg & MV_TDMA_CONTROL_ACT) ? "+" : "-");

return (const char *)buf;
}

_STATIC const char *
s_xpsecintr(uint32_t v)
{
static char buf[160];

snprintf(buf, sizeof(buf),
"%sAuth %sDES %sAES-ENC %sAES-DEC %sENC %sSA %sAccAndTDMA "
"%sTDMAComp %sTDMAOwn %sAccAndTDMA_Cont",
(v & MVXPSEC_INT_AUTH) ? "+" : "-",
(v & MVXPSEC_INT_DES) ? "+" : "-",
(v & MVXPSEC_INT_AES_ENC) ? "+" : "-",
(v & MVXPSEC_INT_AES_DEC) ? "+" : "-",
(v & MVXPSEC_INT_ENC) ? "+" : "-",
(v & MVXPSEC_INT_SA) ? "+" : "-",
(v & MVXPSEC_INT_ACCTDMA) ? "+" : "-",
(v & MVXPSEC_INT_TDMA_COMP) ? "+" : "-",
(v & MVXPSEC_INT_TDMA_OWN) ? "+" : "-",
(v & MVXPSEC_INT_ACCTDMA_CONT) ? "+" : "-");

return (const char *)buf;
}

STATIC const char *
s_ctlalg(uint32_t alg)
{
switch (alg) {
case CRYPTO_SHA1_HMAC_96:
return "HMAC-SHA1-96";
case CRYPTO_SHA1_HMAC:
return "HMAC-SHA1";
case CRYPTO_SHA1:
return "SHA1";
case CRYPTO_MD5_HMAC_96:
return "HMAC-MD5-96";
case CRYPTO_MD5_HMAC:
return "HMAC-MD5";
case CRYPTO_MD5:
return "MD5";
case CRYPTO_DES_CBC:
return "DES-CBC";
case CRYPTO_3DES_CBC:
return "3DES-CBC";
case CRYPTO_AES_CBC:
return "AES-CBC";
default:
break;
}

return "Unknown";
}

STATIC const char *
s_xpsec_op(uint32_t reg)
{
reg &= MV_ACC_CRYPTO_OP_MASK;
switch (reg) {
case MV_ACC_CRYPTO_OP_ENC:
return "ENC";
case MV_ACC_CRYPTO_OP_MAC:
return "MAC";
case MV_ACC_CRYPTO_OP_ENCMAC:
return "ENC-MAC";
case MV_ACC_CRYPTO_OP_MACENC:
return "MAC-ENC";
default:
break;
}

return "Unknown";
}

STATIC const char *
s_xpsec_enc(uint32_t alg)
{
alg <<= MV_ACC_CRYPTO_ENC_SHIFT;
switch (alg) {
case MV_ACC_CRYPTO_ENC_DES:
return "DES";
case MV_ACC_CRYPTO_ENC_3DES:
return "3DES";
case MV_ACC_CRYPTO_ENC_AES:
return "AES";
default:
break;
}

return "Unknown";
}

STATIC const char *
s_xpsec_mac(uint32_t alg)
{
alg <<= MV_ACC_CRYPTO_MAC_SHIFT;
switch (alg) {
case MV_ACC_CRYPTO_MAC_NONE:
return "Disabled";
case MV_ACC_CRYPTO_MAC_MD5:
return "MD5";
case MV_ACC_CRYPTO_MAC_SHA1:
return "SHA1";
case MV_ACC_CRYPTO_MAC_HMAC_MD5:
return "HMAC-MD5";
case MV_ACC_CRYPTO_MAC_HMAC_SHA1:
return "HMAC-SHA1";
default:
break;
}

return "Unknown";
}

STATIC const char *
s_xpsec_frag(uint32_t frag)
{
frag <<= MV_ACC_CRYPTO_FRAG_SHIFT;
switch (frag) {
case MV_ACC_CRYPTO_NOFRAG:
return "NoFragment";
case MV_ACC_CRYPTO_FRAG_FIRST:
return "FirstFragment";
case MV_ACC_CRYPTO_FRAG_MID:
return "MiddleFragment";
case MV_ACC_CRYPTO_FRAG_LAST:
return "LastFragment";
default:
break;
}

return "Unknown";
}

#ifdef MVXPSEC_DEBUG
void
mvxpsec_dump_reg(struct mvxpsec_softc *sc)
{
uint32_t reg;
int i;

if ((mvxpsec_debug & MVXPSEC_DEBUG_DESC) == 0)
return;

printf("--- Interrupt Registers ---\n");
reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE);
printf("MVXPSEC INT CAUSE: 0x%08x\n", reg);
printf("MVXPSEC INT CAUSE: %s\n", s_xpsecintr(reg));
reg = MVXPSEC_READ(sc, MVXPSEC_INT_MASK);
printf("MVXPSEC INT MASK: 0x%08x\n", reg);
printf("MVXPSEC INT MASKE: %s\n", s_xpsecintr(reg));

printf("--- DMA Configuration Registers ---\n");
for (i = 0; i < MV_TDMA_NWINDOW; i++) {
reg = MVXPSEC_READ(sc, MV_TDMA_BAR(i));
printf("TDMA BAR%d: 0x%08x\n", i, reg);
reg = MVXPSEC_READ(sc, MV_TDMA_ATTR(i));
printf("TDMA ATTR%d: 0x%08x\n", i, reg);
printf(" -> %s\n", s_winreg(reg));
}

printf("--- DMA Control Registers ---\n");

reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
printf("TDMA CONTROL: 0x%08x\n", reg);
printf(" -> %s\n", s_ctrlreg(reg));

printf("--- DMA Current Command Descriptors ---\n");

reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE);
printf("TDMA ERR CAUSE: 0x%08x\n", reg);

reg = MVXPSEC_READ(sc, MV_TDMA_ERR_MASK);
printf("TDMA ERR MASK: 0x%08x\n", reg);

reg = MVXPSEC_READ(sc, MV_TDMA_CNT);
printf("TDMA DATA OWNER: %s\n",
(reg & MV_TDMA_CNT_OWN) ? "DMAC" : "CPU");
printf("TDMA DATA COUNT: %d(0x%x)\n",
(reg & ~MV_TDMA_CNT_OWN), (reg & ~MV_TDMA_CNT_OWN));

reg = MVXPSEC_READ(sc, MV_TDMA_SRC);
printf("TDMA DATA SRC: 0x%08x\n", reg);

reg = MVXPSEC_READ(sc, MV_TDMA_DST);
printf("TDMA DATA DST: 0x%08x\n", reg);

reg = MVXPSEC_READ(sc, MV_TDMA_NXT);
printf("TDMA DATA NXT: 0x%08x\n", reg);

reg = MVXPSEC_READ(sc, MV_TDMA_CUR);
printf("TDMA DATA CUR: 0x%08x\n", reg);

printf("--- ACC Command Register ---\n");
reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
printf("ACC COMMAND: 0x%08x\n", reg);
printf("ACC: %sACT %sSTOP\n",
(reg & MV_ACC_COMMAND_ACT) ? "+" : "-",
(reg & MV_ACC_COMMAND_STOP) ? "+" : "-");

reg = MVXPSEC_READ(sc, MV_ACC_CONFIG);
printf("ACC CONFIG: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_ACC_DESC);
printf("ACC DESC: 0x%08x\n", reg);

printf("--- DES Key Register ---\n");
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0L);
printf("DES KEY0 Low: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0H);
printf("DES KEY0 High: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1L);
printf("DES KEY1 Low: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1H);
printf("DES KEY1 High: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2L);
printf("DES KEY2 Low: 0x%08x\n", reg);
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2H);
printf("DES KEY2 High: 0x%08x\n", reg);

printf("--- AES Key Register ---\n");
for (i = 0; i < 8; i++) {
reg = MVXPSEC_READ(sc, MV_CE_AES_EKEY(i));
printf("AES ENC KEY COL%d: %08x\n", i, reg);
}
for (i = 0; i < 8; i++) {
reg = MVXPSEC_READ(sc, MV_CE_AES_DKEY(i));
printf("AES DEC KEY COL%d: %08x\n", i, reg);
}

return;
}

STATIC void
mvxpsec_dump_sram(const char *name, struct mvxpsec_softc *sc, size_t len)
{
uint32_t reg;

if (sc->sc_sram_va == NULL)
return;

if (len == 0) {
printf("\n%s NO DATA(len=0)\n", name);
return;
}
else if (len > MV_ACC_SRAM_SIZE)
len = MV_ACC_SRAM_SIZE;

mutex_enter(&sc->sc_dma_mtx);
reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL);
if (reg & MV_TDMA_CONTROL_ACT) {
printf("TDMA is active, cannot access SRAM\n");
mutex_exit(&sc->sc_dma_mtx);
return;
}
reg = MVXPSEC_READ(sc, MV_ACC_COMMAND);
if (reg & MV_ACC_COMMAND_ACT) {
printf("SA is active, cannot access SRAM\n");
mutex_exit(&sc->sc_dma_mtx);
return;
}

printf("%s: dump SRAM, %zu bytes\n", name, len);
mvxpsec_dump_data(name, sc->sc_sram_va, len);
mutex_exit(&sc->sc_dma_mtx);
return;
}

_STATIC void
mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *dh)
{
struct mvxpsec_descriptor *d =
(struct mvxpsec_descriptor *)dh->_desc;

printf("--- DMA Command Descriptor ---\n");
printf("DESC: VA=%p PA=0x%08x\n",
d, (uint32_t)dh->phys_addr);
printf("DESC: WORD0 = 0x%08x\n", d->tdma_word0);
printf("DESC: SRC = 0x%08x\n", d->tdma_src);
printf("DESC: DST = 0x%08x\n", d->tdma_dst);
printf("DESC: NXT = 0x%08x\n", d->tdma_nxt);

return;
}

STATIC void
mvxpsec_dump_data(const char *name, void *p, size_t len)
{
uint8_t *data = p;
off_t off;

printf("%s: dump %p, %zu bytes", name, p, len);
if (p == NULL || len == 0) {
printf("\n%s: NO DATA\n", name);
return;
}
for (off = 0; off < len; off++) {
if ((off % 16) == 0) {
printf("\n%s: 0x%08x:", name, (uint32_t)off);
}
if ((off % 4) == 0) {
printf(" ");
}
printf("%02x", data[off]);
}
printf("\n");

return;
}

_STATIC void
mvxpsec_dump_packet(const char *name, struct mvxpsec_packet *mv_p)
{
struct mvxpsec_softc *sc = mv_p->mv_s->sc;

printf("%s: packet_data:\n", name);
mvxpsec_dump_packet_data(name, mv_p);

printf("%s: SRAM:\n", name);
mvxpsec_dump_sram(name, sc, 2000);

printf("%s: packet_descriptor:\n", name);
mvxpsec_dump_packet_desc(name, mv_p);
}

_STATIC void
mvxpsec_dump_packet_data(const char *name, struct mvxpsec_packet *mv_p)
{
static char buf[1500];
int len;

if (mv_p->data_type == MVXPSEC_DATA_MBUF) {
struct mbuf *m;

m = mv_p->data.mbuf;
len = m->m_pkthdr.len;
if (len > sizeof(buf))
len = sizeof(buf);
m_copydata(m, 0, len, buf);
}
else if (mv_p->data_type == MVXPSEC_DATA_UIO) {
struct uio *uio;

uio = mv_p->data.uio;
len = uio->uio_resid;
if (len > sizeof(buf))
len = sizeof(buf);
cuio_copydata(uio, 0, len, buf);
}
else if (mv_p->data_type == MVXPSEC_DATA_RAW) {
len = mv_p->data_len;
if (len > sizeof(buf))
len = sizeof(buf);
memcpy(buf, mv_p->data.raw, len);
}
else
return;
mvxpsec_dump_data(name, buf, len);

return;
}

_STATIC void
mvxpsec_dump_packet_desc(const char *name, struct mvxpsec_packet *mv_p)
{
uint32_t *words;

if (mv_p == NULL)
return;

words = &mv_p->pkt_header.desc.acc_desc_dword0;
mvxpsec_dump_acc_config(name, words[0]);
mvxpsec_dump_acc_encdata(name, words[1], words[2]);
mvxpsec_dump_acc_enclen(name, words[2]);
mvxpsec_dump_acc_enckey(name, words[3]);
mvxpsec_dump_acc_enciv(name, words[4]);
mvxpsec_dump_acc_macsrc(name, words[5]);
mvxpsec_dump_acc_macdst(name, words[6]);
mvxpsec_dump_acc_maciv(name, words[7]);

return;
}

_STATIC void
mvxpsec_dump_acc_config(const char *name, uint32_t w)
{
/* SA: Dword 0 */
printf("%s: Dword0=0x%08x\n", name, w);
printf("%s: OP = %s\n", name,
s_xpsec_op(MV_ACC_CRYPTO_OP(w)));
printf("%s: MAC = %s\n", name,
s_xpsec_mac(MV_ACC_CRYPTO_MAC(w)));
printf("%s: MAC_LEN = %s\n", name,
w & MV_ACC_CRYPTO_MAC_96 ? "96-bit" : "full-bit");
printf("%s: ENC = %s\n", name,
s_xpsec_enc(MV_ACC_CRYPTO_ENC(w)));
printf("%s: DIR = %s\n", name,
w & MV_ACC_CRYPTO_DECRYPT ? "decryption" : "encryption");
printf("%s: CHAIN = %s\n", name,
w & MV_ACC_CRYPTO_CBC ? "CBC" : "ECB");
printf("%s: 3DES = %s\n", name,
w & MV_ACC_CRYPTO_3DES_EDE ? "EDE" : "EEE");
printf("%s: FRAGMENT = %s\n", name,
s_xpsec_frag(MV_ACC_CRYPTO_FRAG(w)));
return;
}

STATIC void
mvxpsec_dump_acc_encdata(const char *name, uint32_t w, uint32_t w2)
{
/* SA: Dword 1 */
printf("%s: Dword1=0x%08x\n", name, w);
printf("%s: ENC SRC = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
printf("%s: ENC DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
printf("%s: ENC RANGE = 0x%x - 0x%x\n", name,
MV_ACC_DESC_GET_VAL_1(w),
MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_1(w2) - 1);
return;
}

STATIC void
mvxpsec_dump_acc_enclen(const char *name, uint32_t w)
{
/* SA: Dword 2 */
printf("%s: Dword2=0x%08x\n", name, w);
printf("%s: ENC LEN = %d\n", name,
MV_ACC_DESC_GET_VAL_1(w));
return;
}

STATIC void
mvxpsec_dump_acc_enckey(const char *name, uint32_t w)
{
/* SA: Dword 3 */
printf("%s: Dword3=0x%08x\n", name, w);
printf("%s: EKEY = 0x%x\n", name,
MV_ACC_DESC_GET_VAL_1(w));
return;
}

STATIC void
mvxpsec_dump_acc_enciv(const char *name, uint32_t w)
{
/* SA: Dword 4 */
printf("%s: Dword4=0x%08x\n", name, w);
printf("%s: EIV = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
printf("%s: EIV_BUF = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w));
return;
}

STATIC void
mvxpsec_dump_acc_macsrc(const char *name, uint32_t w)
{
/* SA: Dword 5 */
printf("%s: Dword5=0x%08x\n", name, w);
printf("%s: MAC_SRC = 0x%x\n", name,
MV_ACC_DESC_GET_VAL_1(w));
printf("%s: MAC_TOTAL_LEN = %d\n", name,
MV_ACC_DESC_GET_VAL_3(w));
printf("%s: MAC_RANGE = 0x%0x - 0x%0x\n", name,
MV_ACC_DESC_GET_VAL_1(w),
MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_3(w) - 1);
return;
}

STATIC void
mvxpsec_dump_acc_macdst(const char *name, uint32_t w)
{
/* SA: Dword 6 */
printf("%s: Dword6=0x%08x\n", name, w);
printf("%s: MAC_DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w));
printf("%s: MAC_BLOCK_LEN = %d\n", name,
MV_ACC_DESC_GET_VAL_2(w));
return;
}

STATIC void
mvxpsec_dump_acc_maciv(const char *name, uint32_t w)
{
/* SA: Dword 7 */
printf("%s: Dword7=0x%08x\n", name, w);
printf("%s: MAC_INNER_IV = 0x%x\n", name,
MV_ACC_DESC_GET_VAL_1(w));
printf("%s: MAC_OUTER_IV = 0x%x\n", name,
MV_ACC_DESC_GET_VAL_2(w));
return;
}
#endif