* Copyright (c) 2019 Internet Initiative Japan, Inc.

/* $NetBSD: qat_ae.c,v 1.2 2021/12/05 07:28:20 msaitoh Exp $ */

/*
* Copyright (c) 2019 Internet Initiative Japan, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.2 2021/12/05 07:28:20 msaitoh Exp $");

#include <sys/param.h>
#include <sys/systm.h>

#include <dev/firmload.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#include "qatreg.h"
#include "qatvar.h"
#include "qat_aevar.h"

int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
uint32_t);
int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t,
uint32_t *);
int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
uint32_t);
void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t,
bus_size_t, uint32_t);
int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t,
bus_size_t, uint32_t *);

u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short);
int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, uint32_t);
int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, uint32_t *);
int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, uint32_t);
int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, uint32_t);
int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, uint32_t);
int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short,
u_short *, u_char *);
int qat_aereg_abs_data_write(struct qat_softc *, u_char,
enum aereg_type, u_short, uint32_t);

void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int);
void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int);
void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char);
void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char);
void qat_ae_write_lm_mode(struct qat_softc *, u_char,
enum aereg_type, u_char);
void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char,
u_char);
void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char);
int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int,
u_int);

enum qat_ae_status
qat_ae_get_status(struct qat_softc *, u_char);
int qat_ae_is_active(struct qat_softc *, u_char);
int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int);

int qat_ae_clear_reset(struct qat_softc *);
int qat_ae_check(struct qat_softc *);
int qat_ae_reset_timestamp(struct qat_softc *);
void qat_ae_clear_xfer(struct qat_softc *);
int qat_ae_clear_gprs(struct qat_softc *);

void qat_ae_get_shared_ustore_ae(u_char, u_char *);
u_int qat_ae_ucode_parity64(uint64_t);
uint64_t qat_ae_ucode_set_ecc(uint64_t);
int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int,
const uint64_t *);
int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int,
uint64_t *);
u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *);
int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char,
uint64_t *, u_int, int, u_int, u_int *);
int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char,
int *, uint64_t *, u_int,
u_int *, u_int *, u_int *, u_int *, u_int *);
int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char,
u_int, u_int, u_int, u_int, u_int);
int qat_ae_get_inst_num(int);
int qat_ae_batch_put_lm(struct qat_softc *, u_char,
struct qat_ae_batch_init_list *, size_t);
int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int);

u_int qat_aefw_csum(char *, int);
const char * qat_aefw_uof_string(struct qat_softc *, size_t);
struct uof_chunk_hdr *
qat_aefw_uof_find_chunk(struct qat_softc *, const char *,
struct uof_chunk_hdr *);

int qat_aefw_load_mof(struct qat_softc *);
int qat_aefw_load_mmp(struct qat_softc *);

int qat_aefw_mof_find_uof0(struct qat_softc *,
struct mof_uof_hdr *, struct mof_uof_chunk_hdr *,
u_int, size_t, const char *,
size_t *, void **);
int qat_aefw_mof_find_uof(struct qat_softc *);
int qat_aefw_mof_parse(struct qat_softc *);

int qat_aefw_uof_parse_image(struct qat_softc *,
struct qat_uof_image *, struct uof_chunk_hdr *uch);
int qat_aefw_uof_parse_images(struct qat_softc *);
int qat_aefw_uof_parse(struct qat_softc *);

int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t,
struct qat_dmamem *);
int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *);
int qat_aefw_suof_load(struct qat_softc *sc,
struct qat_dmamem *dma);
int qat_aefw_suof_parse_image(struct qat_softc *,
struct qat_suof_image *, struct suof_chunk_hdr *);
int qat_aefw_suof_parse(struct qat_softc *);
int qat_aefw_suof_write(struct qat_softc *);

int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *,
struct qat_uof_image *);
int qat_aefw_uof_init_ae(struct qat_softc *, u_char);
int qat_aefw_uof_init(struct qat_softc *);

int qat_aefw_init_memory_one(struct qat_softc *,
struct uof_init_mem *);
void qat_aefw_free_lm_init(struct qat_softc *, u_char);
int qat_aefw_init_ustore(struct qat_softc *);
int qat_aefw_init_reg(struct qat_softc *, u_char, u_char,
enum aereg_type, u_short, u_int);
int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char,
struct qat_uof_image *);
int qat_aefw_init_memory(struct qat_softc *);
int qat_aefw_init_globals(struct qat_softc *);
uint64_t qat_aefw_get_uof_inst(struct qat_softc *,
struct qat_uof_page *, u_int);
int qat_aefw_do_pagein(struct qat_softc *, u_char,
struct qat_uof_page *);
int qat_aefw_uof_write_one(struct qat_softc *, struct qat_uof_image *);
int qat_aefw_uof_write(struct qat_softc *);

int
qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
uint32_t value)
{
int times = TIMEOUT_AE_CSR;

do {
qat_ae_local_write_4(sc, ae, offset, value);
if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
LOCAL_CSR_STATUS_STATUS) == 0)
return 0;

} while (times--);

aprint_error_dev(sc->sc_dev,
"couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
return EFAULT;
}

int
qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
uint32_t *value)
{
int times = TIMEOUT_AE_CSR;
uint32_t v;

do {
v = qat_ae_local_read_4(sc, ae, offset);
if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
LOCAL_CSR_STATUS_STATUS) == 0) {
*value = v;
return 0;
}
} while (times--);

aprint_error_dev(sc->sc_dev,
"couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
return EFAULT;
}

void
qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask,
bus_size_t offset, uint32_t value)
{
int ctx;
uint32_t ctxptr;

KASSERT(offset == CTX_FUTURE_COUNT_INDIRECT ||
offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
offset == CTX_STS_INDIRECT ||
offset == CTX_WAKEUP_EVENTS_INDIRECT ||
offset == CTX_SIG_EVENTS_INDIRECT ||
offset == LM_ADDR_0_INDIRECT ||
offset == LM_ADDR_1_INDIRECT ||
offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
if ((ctx_mask & (1 << ctx)) == 0)
continue;
qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
qat_ae_write_4(sc, ae, offset, value);
}
qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
}

int
qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx,
bus_size_t offset, uint32_t *value)
{
int error;
uint32_t ctxptr;

KASSERT(offset == CTX_FUTURE_COUNT_INDIRECT ||
offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
offset == CTX_STS_INDIRECT ||
offset == CTX_WAKEUP_EVENTS_INDIRECT ||
offset == CTX_SIG_EVENTS_INDIRECT ||
offset == LM_ADDR_0_INDIRECT ||
offset == LM_ADDR_1_INDIRECT ||
offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

/* save the ctx ptr */
qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
(ctx & CSR_CTX_POINTER_CONTEXT))
qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);

error = qat_ae_read_4(sc, ae, offset, value);

/* restore ctx ptr */
if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
(ctx & CSR_CTX_POINTER_CONTEXT))
qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);

return error;
}

u_short
qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg)
{
u_short addr;

switch (regtype) {
case AEREG_GPA_ABS:
case AEREG_GPB_ABS:
addr = (reg & 0x7f) | 0x80;
break;
case AEREG_GPA_REL:
case AEREG_GPB_REL:
addr = reg & 0x1f;
break;
case AEREG_SR_RD_REL:
case AEREG_SR_WR_REL:
case AEREG_SR_REL:
addr = 0x180 | (reg & 0x1f);
break;
case AEREG_SR_INDX:
addr = 0x140 | ((reg & 0x3) << 1);
break;
case AEREG_DR_RD_REL:
case AEREG_DR_WR_REL:
case AEREG_DR_REL:
addr = 0x1c0 | (reg & 0x1f);
break;
case AEREG_DR_INDX:
addr = 0x100 | ((reg & 0x3) << 1);
break;
case AEREG_NEIGH_INDX:
addr = 0x241 | ((reg & 0x3) << 1);
break;
case AEREG_NEIGH_REL:
addr = 0x280 | (reg & 0x1f);
break;
case AEREG_LMEM0:
addr = 0x200;
break;
case AEREG_LMEM1:
addr = 0x220;
break;
case AEREG_NO_DEST:
addr = 0x300 | (reg & 0xff);
break;
default:
addr = AEREG_BAD_REGADDR;
break;
}
return (addr);
}

int
qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx,
enum aereg_type regtype, u_short relreg, uint32_t value)
{
uint16_t srchi, srclo, destaddr, data16hi, data16lo;
uint64_t inst[] = {
0x0F440000000ull, /* immed_w1[reg, val_hi16] */
0x0F040000000ull, /* immed_w0[reg, val_lo16] */
0x0F0000C0300ull, /* nop */
0x0E000010000ull /* ctx_arb[kill] */
};
const int ninst = __arraycount(inst);
const int imm_w1 = 0, imm_w0 = 1;
unsigned int ctxen;
uint16_t mask;

/* This logic only works for GPRs and LM index registers,
not NN or XFER registers! */
KASSERT(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) {
/* determine the context mode */
qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
/* 4-ctx mode */
if (ctx & 0x1)
return EINVAL;
mask = 0x1f;
} else {
/* 8-ctx mode */
mask = 0x0f;
}
if (relreg & ~mask)
return EINVAL;
}
if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
AEREG_BAD_REGADDR) {
return EINVAL;
}

data16lo = 0xffff & value;
data16hi = 0xffff & (value >> 16);
srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
(uint16_t)(0xff & data16hi));
srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
(uint16_t)(0xff & data16lo));

switch (regtype) {
case AEREG_GPA_REL: /* A rel source */
inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
((srchi & 0x3ff) << 10) | (destaddr & 0x3ff);
inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
((srclo & 0x3ff) << 10) | (destaddr & 0x3ff);
break;
default:
inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
((destaddr & 0x3ff) << 10) | (srchi & 0x3ff);
inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
((destaddr & 0x3ff) << 10) | (srclo & 0x3ff);
break;
}

return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL);
}

int
qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx,
enum aereg_type regtype, u_short relreg, uint32_t *value)
{
uint64_t inst, savucode;
uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi;
u_int uaddr, ustore_addr;
int error;
u_short mask, regaddr;
u_char nae;

KASSERT(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL ||
regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL ||
regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) ||
(regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) ||
(regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL))
{
/* determine the context mode */
qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
/* 4-ctx mode */
if (ctx & 0x1)
return EINVAL;
mask = 0x1f;
} else {
/* 8-ctx mode */
mask = 0x0f;
}
if (relreg & ~mask)
return EINVAL;
}
if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
AEREG_BAD_REGADDR) {
return EINVAL;
}

/* instruction -- alu[--, --, B, reg] */
switch (regtype) {
case AEREG_GPA_REL:
/* A rel source */
inst = 0xA070000000ull | (regaddr & 0x3ff);
break;
default:
inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10));
break;
}

/* backup shared control store bit, and force AE to
* none-shared mode before executing ucode snippet */
qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
if (misc & AE_MISC_CONTROL_SHARE_CS) {
qat_ae_get_shared_ustore_ae(ae, &nae);
if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae))
return EBUSY;
}

nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

/* read current context */
qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
/* prevent clearing the W1C bits: the breakpoint bit,
ECC error bit, and Parity error bit */
ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

/* change the context */
if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO))
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
ctx & ACTIVE_CTX_STATUS_ACNO);
/* save a ustore location */
if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) {
/* restore AE_MISC_CONTROL csr */
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

/* restore the context */
if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
savctx & ACTIVE_CTX_STATUS_ACNO);
}
qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

return (error);
}

/* turn off ustore parity */
qat_ae_write_4(sc, ae, CTX_ENABLES,
ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE));

/* save ustore-addr csr */
qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

/* write the ALU instruction to ustore, enable ecs bit */
uaddr = 0 | USTORE_ADDRESS_ECS;

/* set the uaddress */
qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
inst = qat_ae_ucode_set_ecc(inst);

ulo = (uint32_t)(inst & 0xffffffff);
uhi = (uint32_t)(inst >> 32);

qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);

/* this will auto increment the address */
qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);

/* set the uaddress */
qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

/* delay for at least 8 cycles */
qat_ae_wait_num_cycles(sc, ae, 0x8, 0);

/* read ALU output -- the instruction should have been executed
prior to clearing the ECS in putUwords */
qat_ae_read_4(sc, ae, ALU_OUT, value);

/* restore ustore-addr csr */
qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

/* restore the ustore */
error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode);

/* restore the context */
if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
savctx & ACTIVE_CTX_STATUS_ACNO);
}

qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

/* restore AE_MISC_CONTROL csr */
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

return error;
}

int
qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
enum aereg_type regtype, u_short relreg, uint32_t value)
{
bus_size_t addr;
int error;
uint32_t ctxen;
u_short mask;
u_short dr_offset;

KASSERT(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL ||
regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL);

QAT_YIELD();

error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
if (ctx & 0x1) {
aprint_error_dev(sc->sc_dev,
"bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx);
return EINVAL;
}
mask = 0x1f;
dr_offset = 0x20;

} else {
mask = 0x0f;
dr_offset = 0x10;
}

if (relreg & ~mask)
return EINVAL;

addr = relreg + (ctx << 0x5);

switch (regtype) {
case AEREG_SR_REL:
case AEREG_SR_RD_REL:
qat_ae_xfer_write_4(sc, ae, addr, value);
break;
case AEREG_DR_REL:
case AEREG_DR_RD_REL:
qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value);
break;
default:
error = EINVAL;
}

return error;
}

int
qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
enum aereg_type regtype, u_short relreg, uint32_t value)
{

panic("notyet");

return 0;
}

int
qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx,
enum aereg_type regtype, u_short relreg, uint32_t value)
{

panic("notyet");

return 0;
}

int
qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae,
u_short absreg, u_short *relreg, u_char *ctx)
{
uint32_t ctxen;

qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
/* 4-ctx mode */
*relreg = absreg & 0x1f;
*ctx = (absreg >> 0x4) & 0x6;
} else {
/* 8-ctx mode */
*relreg = absreg & 0x0f;
*ctx = (absreg >> 0x4) & 0x7;
}

return 0;
}

int
qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae,
enum aereg_type regtype, u_short absreg, uint32_t value)
{
int error;
u_short relreg;
u_char ctx;

qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx);

switch (regtype) {
case AEREG_GPA_ABS:
KASSERT(absreg < MAX_GPR_REG);
error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL,
relreg, value);
break;
case AEREG_GPB_ABS:
KASSERT(absreg < MAX_GPR_REG);
error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL,
relreg, value);
break;
case AEREG_DR_RD_ABS:
KASSERT(absreg < MAX_XFER_REG);
error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL,
relreg, value);
break;
case AEREG_SR_RD_ABS:
KASSERT(absreg < MAX_XFER_REG);
error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL,
relreg, value);
break;
case AEREG_DR_WR_ABS:
KASSERT(absreg < MAX_XFER_REG);
error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL,
relreg, value);
break;
case AEREG_SR_WR_ABS:
KASSERT(absreg < MAX_XFER_REG);
error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL,
relreg, value);
break;
case AEREG_NEIGH_ABS:
KASSERT(absreg < MAX_NN_REG);
if (absreg >= MAX_NN_REG)
return EINVAL;
error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL,
relreg, value);
break;
default:
panic("Invalid Register Type");
}

return error;
}

void
qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
uint32_t ctxen;

qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
ctx_mask &= 0x55;
} else {
ctx_mask &= 0xff;
}

ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE);
qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

void
qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
uint32_t ctxen;

qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE));
qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

void
qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
uint32_t val, nval;

qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
val &= CTX_ENABLES_IGNORE_W1C_MASK;

if (mode == 4)
nval = val | CTX_ENABLES_INUSE_CONTEXTS;
else
nval = val & ~CTX_ENABLES_INUSE_CONTEXTS;

if (val != nval)
qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

void
qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
uint32_t val, nval;

qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
val &= CTX_ENABLES_IGNORE_W1C_MASK;

if (mode)
nval = val | CTX_ENABLES_NN_MODE;
else
nval = val & ~CTX_ENABLES_NN_MODE;

if (val != nval)
qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

void
qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae,
enum aereg_type lm, u_char mode)
{
uint32_t val, nval;
uint32_t bit;

qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
val &= CTX_ENABLES_IGNORE_W1C_MASK;

switch (lm) {
case AEREG_LMEM0:
bit = CTX_ENABLES_LMADDR_0_GLOBAL;
break;
case AEREG_LMEM1:
bit = CTX_ENABLES_LMADDR_1_GLOBAL;
break;
default:
panic("invalid lmem reg type");
break;
}

if (mode)
nval = val | bit;
else
nval = val & ~bit;

if (val != nval)
qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

void
qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode)
{
uint32_t val, nval;

qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);

if (mode == 1)
nval = val | AE_MISC_CONTROL_SHARE_CS;
else
nval = val & ~AE_MISC_CONTROL_SHARE_CS;

if (val != nval)
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval);
}

void
qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
u_char nae;

qat_ae_get_shared_ustore_ae(ae, &nae);

qat_ae_write_shared_cs_mode0(sc, ae, mode);

if ((sc->sc_ae_mask & (1 << nae))) {
qat_ae_write_shared_cs_mode0(sc, nae, mode);
}
}

int
qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae,
u_int reload_size, int shared_mode, u_int ustore_dram_addr)
{
uint32_t val, cs_reload;

switch (reload_size) {
case 0:
cs_reload = 0x0;
break;
case QAT_2K:
cs_reload = 0x1;
break;
case QAT_4K:
cs_reload = 0x2;
break;
case QAT_8K:
cs_reload = 0x3;
break;
default:
return EINVAL;
}

if (cs_reload)
QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr;

QAT_AE(sc, ae).qae_reload_size = reload_size;

qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD |
AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS);
val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) |
__SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD);
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

return 0;
}

enum qat_ae_status
qat_ae_get_status(struct qat_softc *sc, u_char ae)
{
int error;
uint32_t val = 0;

error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
if (error || val & CTX_ENABLES_ENABLE)
return QAT_AE_ENABLED;

qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
if (val & ACTIVE_CTX_STATUS_ABO)
return QAT_AE_ACTIVE;

return QAT_AE_DISABLED;
}

int
qat_ae_is_active(struct qat_softc *sc, u_char ae)
{
uint32_t val;

if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
return 1;

qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
if (val & ACTIVE_CTX_STATUS_ABO)
return 1;
else
return 0;
}

/* returns 1 if actually waited for specified number of cycles */
int
qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check)
{
uint32_t cnt, actx;
int pcnt, ccnt, elapsed, times;

qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
pcnt = cnt & 0xffff;

times = TIMEOUT_AE_CHECK;
do {
qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
ccnt = cnt & 0xffff;

elapsed = ccnt - pcnt;
if (elapsed == 0) {
times--;
aprint_debug_dev(sc->sc_dev,
"qat_ae_wait_num_cycles elapsed 0 times %d\n",
times);
}
if (times <= 0) {
aprint_error_dev(sc->sc_dev,
"qat_ae_wait_num_cycles timeout\n");
return -1;
}

if (elapsed < 0)
elapsed += 0x10000;

if (elapsed >= CYCLES_FROM_READY2EXE && check) {
if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS,
&actx) == 0) {
if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
return 0;
}
}
} while (cycles > elapsed);

if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) {
if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
return 0;
}

return 1;
}

int
qat_ae_init(struct qat_softc *sc)
{
int error;
uint32_t mask, val = 0;
u_char ae;

/* XXX adf_initSysMemInfo */

/* XXX Disable clock gating for some chip if debug mode */

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
struct qat_ae *qae = &sc->sc_ae[ae];
if (!(mask & 1))
continue;

qae->qae_ustore_size = USTORE_SIZE;

qae->qae_free_addr = 0;
qae->qae_free_size = USTORE_SIZE;
qae->qae_live_ctx_mask = AE_ALL_CTX;
qae->qae_ustore_dram_addr = 0;
qae->qae_reload_size = 0;
}

/* XXX Enable attention interrupt */

error = qat_ae_clear_reset(sc);
if (error)
return error;

qat_ae_clear_xfer(sc);

if (!sc->sc_hw.qhw_fw_auth) {
error = qat_ae_clear_gprs(sc);
if (error)
return error;
}

/* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;
qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val);
val |= 0x1;
qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val);
}

error = qat_ae_clear_reset(sc);
if (error)
return error;

/* XXX XXX XXX Clean MMP memory if mem scrub is supported */
/* halMem_ScrubMMPMemory */

return 0;
}

int
qat_ae_start(struct qat_softc *sc)
{
int error;

u_char ae;

for (ae = 0; ae < sc->sc_ae_num; ae++) {
if ((sc->sc_ae_mask & (1 << ae)) == 0)
continue;

error = qat_aefw_start(sc, ae, 0xff);
if (error)
return error;

aprint_verbose_dev(sc->sc_dev, "Started AE %d\n", ae);
}

return 0;
}

int
qat_ae_cluster_intr(void *arg)
{
/* XXX */
printf("qat_ae_cluster_intr\n");

return 1;
}

int
qat_ae_clear_reset(struct qat_softc *sc)
{
int error;
uint32_t times, reset, clock, reg, mask;
u_char ae;

reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK));
reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK));
times = TIMEOUT_AE_RESET;
do {
qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset);
if ((times--) == 0) {
aprint_error_dev(sc->sc_dev, "couldn't reset AEs\n");
return EBUSY;
}
reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
} while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) |
__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK))
& reg);

/* Enable clock for AE and QAT */
clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN);
clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK);
clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK);
qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock);

error = qat_ae_check(sc);
if (error)
return error;

/*
* Set undefined power-up/reset states to reasonable default values...
* just to make sure we're starting from a known point
*/
for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;

/* init the ctx_enable */
qat_ae_write_4(sc, ae, CTX_ENABLES,
CTX_ENABLES_INIT);

/* initialize the PCs */
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_STS_INDIRECT,
UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

/* init the ctx_arb */
qat_ae_write_4(sc, ae, CTX_ARB_CNTL,
CTX_ARB_CNTL_INIT);

/* enable cc */
qat_ae_write_4(sc, ae, CC_ENABLE,
CC_ENABLE_INIT);
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_WAKEUP_EVENTS_INDIRECT,
CTX_WAKEUP_EVENTS_INDIRECT_INIT);
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_SIG_EVENTS_INDIRECT,
CTX_SIG_EVENTS_INDIRECT_INIT);
}

if ((sc->sc_ae_mask != 0) &&
sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) {
/* XXX XXX XXX init eSram only when this is boot time */
}

if ((sc->sc_ae_mask != 0) &&
sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) {
/* XXX XXX XXX wait shram to complete initialization */
}

qat_ae_reset_timestamp(sc);

return 0;
}

int
qat_ae_check(struct qat_softc *sc)
{
int error, times, ae;
uint32_t cnt, pcnt, mask;

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;

times = TIMEOUT_AE_CHECK;
error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
if (error) {
aprint_error_dev(sc->sc_dev,
"couldn't access AE %d CSR\n", ae);
return error;
}
pcnt = cnt & 0xffff;

while (1) {
error = qat_ae_read_4(sc, ae,
PROFILE_COUNT, &cnt);
if (error) {
aprint_error_dev(sc->sc_dev,
"couldn't access AE %d CSR\n", ae);
return error;
}
cnt &= 0xffff;
if (cnt == pcnt)
times--;
else
break;
if (times <= 0) {
aprint_error_dev(sc->sc_dev,
"AE %d CSR is useless\n", ae);
return EFAULT;
}
}
}

return 0;
}

int
qat_ae_reset_timestamp(struct qat_softc *sc)
{
uint32_t misc, mask;
u_char ae;

/* stop the timestamp timers */
misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC);
if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) {
qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN));
}

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;
qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0);
qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0);
}

/* start timestamp timers */
qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN);

return 0;
}

void
qat_ae_clear_xfer(struct qat_softc *sc)
{
u_int mask, reg;
u_char ae;

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;

for (reg = 0; reg < MAX_GPR_REG; reg++) {
qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS,
reg, 0);
qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS,
reg, 0);
}
}
}

int
qat_ae_clear_gprs(struct qat_softc *sc)
{
uint32_t val;
uint32_t saved_ctx = 0;
int times = TIMEOUT_AE_CHECK, rv;
u_char ae;
u_int mask;

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;

/* turn off share control store bit */
val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
val &= ~AE_MISC_CONTROL_SHARE_CS;
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

/* turn off ucode parity */
/* make sure nn_mode is set to self */
qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
val &= CTX_ENABLES_IGNORE_W1C_MASK;
val |= CTX_ENABLES_NN_MODE;
val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE;
qat_ae_write_4(sc, ae, CTX_ENABLES, val);

/* copy instructions to ustore */
qat_ae_ucode_write(sc, ae, 0, __arraycount(ae_clear_gprs_inst),
ae_clear_gprs_inst);

/* set PC */
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT,
UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

/* save current context */
qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx);
/* change the active context */
/* start the context from ctx 0 */
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0);

/* wakeup-event voluntary */
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_WAKEUP_EVENTS_INDIRECT,
CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
/* clean signals */
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_SIG_EVENTS_INDIRECT, 0);
qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

qat_ae_enable_ctx(sc, ae, AE_ALL_CTX);
}

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;
/* wait for AE to finish */
do {
rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1);
} while (rv && times--);
if (times <= 0) {
aprint_error_dev(sc->sc_dev,
"qat_ae_clear_gprs timeout");
return ETIMEDOUT;
}
qat_ae_disable_ctx(sc, ae, AE_ALL_CTX);
/* change the active context */
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
saved_ctx & ACTIVE_CTX_STATUS_ACNO);
/* init the ctx_enable */
qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT);
/* initialize the PCs */
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
/* init the ctx_arb */
qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT);
/* enable cc */
qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT);
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT);
qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT,
CTX_SIG_EVENTS_INDIRECT_INIT);
}

return 0;
}

void
qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae)
{
if (ae & 0x1)
*nae = ae - 1;
else
*nae = ae + 1;
}

u_int
qat_ae_ucode_parity64(uint64_t ucode)
{

ucode ^= ucode >> 1;
ucode ^= ucode >> 2;
ucode ^= ucode >> 4;
ucode ^= ucode >> 8;
ucode ^= ucode >> 16;
ucode ^= ucode >> 32;

return ((u_int)(ucode & 1));
}

uint64_t
qat_ae_ucode_set_ecc(uint64_t ucode)
{
static const uint64_t
bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL,
bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL,
bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL,
bit6mask=0xdaf69a46910ULL;

/* clear the ecc bits */
ucode &= ~(0x7fULL << USTORE_ECC_BIT_0);

ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) <<
USTORE_ECC_BIT_0;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) <<
USTORE_ECC_BIT_1;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) <<
USTORE_ECC_BIT_2;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) <<
USTORE_ECC_BIT_3;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) <<
USTORE_ECC_BIT_4;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) <<
USTORE_ECC_BIT_5;
ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) <<
USTORE_ECC_BIT_6;

return (ucode);
}

int
qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
const uint64_t *ucode)
{
uint64_t tmp;
uint32_t ustore_addr, ulo, uhi;
int i;

qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
uaddr |= USTORE_ADDRESS_ECS;

qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
for (i = 0; i < ninst; i++) {
tmp = qat_ae_ucode_set_ecc(ucode[i]);
ulo = (uint32_t)(tmp & 0xffffffff);
uhi = (uint32_t)(tmp >> 32);

qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
/* this will auto increment the address */
qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);

QAT_YIELD();
}
qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

return 0;
}

int
qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
uint64_t *ucode)
{
uint32_t misc, ustore_addr, ulo, uhi;
u_int ii;
u_char nae;

if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
return EBUSY;

/* determine whether it neighbour AE runs in shared control store
* status */
qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
if (misc & AE_MISC_CONTROL_SHARE_CS) {
qat_ae_get_shared_ustore_ae(ae, &nae);
if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
return EBUSY;
}

/* if reloadable, then get it all from dram-ustore */
if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD))
panic("notyet"); /* XXX getReloadUwords */

/* disable SHARE_CS bit to workaround silicon bug */
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb);

KASSERT(uaddr + ninst <= USTORE_SIZE);

/* save ustore-addr csr */
qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */
for (ii = 0; ii < ninst; ii++) {
qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

uaddr++;
qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo);
qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi);
ucode[ii] = uhi;
ucode[ii] = (ucode[ii] << 32) | ulo;
}

/* restore SHARE_CS bit to workaround silicon bug */
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

return 0;
}

u_int
qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr,
u_int *value)
{
const uint64_t *inst_arr;
u_int ninst0, curvalue;
int ii, vali, fixup, usize = 0;

if (size == 0)
return 0;

ninst0 = ninst;
vali = 0;
curvalue = value[vali++];

switch (size) {
case 0x1:
inst_arr = ae_inst_1b;
usize = __arraycount(ae_inst_1b);
break;
case 0x2:
inst_arr = ae_inst_2b;
usize = __arraycount(ae_inst_2b);
break;
case 0x3:
inst_arr = ae_inst_3b;
usize = __arraycount(ae_inst_3b);
break;
default:
inst_arr = ae_inst_4b;
usize = __arraycount(ae_inst_4b);
break;
}

fixup = ninst;
for (ii = 0; ii < usize; ii++)
ucode[ninst++] = inst_arr[ii];

INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr));
fixup++;
INSERT_IMMED_GPRA_CONST(ucode[fixup], 0);
fixup++;
INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
fixup++;
INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
/* XXX fixup++ ? */

if (size <= 0x4)
return (ninst - ninst0);

size -= sizeof(u_int);
while (size >= sizeof(u_int)) {
curvalue = value[vali++];
fixup = ninst;
ucode[ninst++] = ae_inst_4b[0x2];
ucode[ninst++] = ae_inst_4b[0x3];
ucode[ninst++] = ae_inst_4b[0x8];
INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
fixup++;
INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
/* XXX fixup++ ? */

addr += sizeof(u_int);
size -= sizeof(u_int);
}
/* call this function recursive when the left size less than 4 */
ninst +=
qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali);

return (ninst - ninst0);
}

int
qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx,
uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles,
u_int *endpc)
{
int error = 0, share_cs = 0;
uint64_t savucode[MAX_EXEC_INST];
uint32_t indr_lm_addr_0, indr_lm_addr_1;
uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1;
uint32_t indr_future_cnt_sig;
uint32_t indr_sig, active_sig;
uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl;
uint32_t misc, nmisc, ctxen;
u_char nae;

KASSERT(ninst <= USTORE_SIZE);

if (qat_ae_is_active(sc, ae))
return EBUSY;

#if 0
printf("%s: ae %d ctx %d ninst %d code 0x%016llx 0x%016llx\n",
__func__, ae, ctx, ninst, ucode[0], ucode[ninst-1]);
#endif

/* save current LM addr */
qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0);
qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1);
qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
&indr_lm_addr_byte_0);
qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
&indr_lm_addr_byte_1);

/* backup shared control store bit, and force AE to
none-shared mode before executing ucode snippet */
qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
if (misc & AE_MISC_CONTROL_SHARE_CS) {
share_cs = 1;
qat_ae_get_shared_ustore_ae(ae, &nae);
if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
return EBUSY;
}
nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

/* save current states: */
if (ninst <= MAX_EXEC_INST) {
error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode);
if (error) {
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
return error;
}
}

/* save wakeup-events */
qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT,
&wakeup_ev);
/* save PC */
qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc);
savpc &= UPC_MASK;

/* save ctx enables */
qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
/* save conditional-code */
qat_ae_read_4(sc, ae, CC_ENABLE, &savcc);
/* save current context */
qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

/* save indirect csrs */
qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
&indr_future_cnt_sig);
qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig);
qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig);

/* turn off ucode parity */
qat_ae_write_4(sc, ae, CTX_ENABLES,
ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE);

/* copy instructions to ustore */
qat_ae_ucode_write(sc, ae, 0, ninst, ucode);
/* set PC */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0);
/* change the active context */
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
ctx & ACTIVE_CTX_STATUS_ACNO);

if (cond_code_off) {
/* disable conditional-code*/
qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff);
}

/* wakeup-event voluntary */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx,
CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);

/* clean signals */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0);
qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

/* enable context */
qat_ae_enable_ctx(sc, ae, 1 << ctx);

/* wait for it to finish */
if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0)
error = ETIMEDOUT;

/* see if we need to get the current PC */
if (endpc != NULL) {
uint32_t ctx_status;

qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
&ctx_status);
*endpc = ctx_status & UPC_MASK;
}
#if 0
{
uint32_t ctx_status;

qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
&ctx_status);
printf("%s: endpc 0x%08x\n", __func__,
ctx_status & UPC_MASK);
}
#endif

/* retore to previous states: */
/* disable context */
qat_ae_disable_ctx(sc, ae, 1 << ctx);
if (ninst <= MAX_EXEC_INST) {
/* instructions */
qat_ae_ucode_write(sc, ae, 0, ninst, savucode);
}
/* wakeup-events */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT,
wakeup_ev);
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc);

/* only restore shared control store bit,
other bit might be changed by AE code snippet */
qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
if (share_cs)
nmisc = misc | AE_MISC_CONTROL_SHARE_CS;
else
nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
/* conditional-code */
qat_ae_write_4(sc, ae, CC_ENABLE, savcc);
/* change the active context */
qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
savctx & ACTIVE_CTX_STATUS_ACNO);
/* restore the nxt ctx to run */
qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
/* restore current LM addr */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT,
indr_lm_addr_0);
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT,
indr_lm_addr_1);
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
indr_lm_addr_byte_0);
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
indr_lm_addr_byte_1);

/* restore indirect csrs */
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
indr_future_cnt_sig);
qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT,
indr_sig);
qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig);

/* ctx-enables */
qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

return error;
}

int
qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx,
int *first_exec, uint64_t *ucode, u_int ninst,
u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1)
{

if (*first_exec) {
qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
*first_exec = 0;
}

return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL);
}

int
qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx,
u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1)
{
qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);

return 0;
}

int
qat_ae_get_inst_num(int lmsize)
{
int ninst, left;

if (lmsize == 0)
return 0;

left = lmsize % sizeof(u_int);

if (left) {
ninst = __arraycount(ae_inst_1b) +
qat_ae_get_inst_num(lmsize - left);
} else {
/* 3 instruction is needed for further code */
ninst = (lmsize - sizeof(u_int)) * 3 / 4 +
__arraycount(ae_inst_4b);
}

return (ninst);
}

int
qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae,
struct qat_ae_batch_init_list *qabi_list, size_t nqabi)
{
struct qat_ae_batch_init *qabi;
size_t alloc_ninst, ninst;
uint64_t *ucode;
u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1;
int insnsz, error = 0, execed = 0, first_exec = 1;

if (SIMPLEQ_FIRST(qabi_list) == NULL)
return 0;

alloc_ninst = uimin(USTORE_SIZE, nqabi);
ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst);

ninst = 0;
SIMPLEQ_FOREACH(qabi, qabi_list, qabi_next) {
insnsz = qat_ae_get_inst_num(qabi->qabi_size);
if (insnsz + ninst > alloc_ninst) {
aprint_debug_dev(sc->sc_dev,
"code page is full, call exection unit\n");
/* add ctx_arb[kill] */
ucode[ninst++] = 0x0E000010000ull;
execed = 1;

error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
&first_exec, ucode, ninst,
&gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
if (error) {
qat_ae_restore_init_lm_gprs(sc, ae, 0,
gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
qat_free_mem(ucode);
return error;
}
/* run microExec to execute the microcode */
ninst = 0;
}
ninst += qat_ae_concat_ucode(ucode, ninst,
qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value);
}

if (ninst > 0) {
ucode[ninst++] = 0x0E000010000ull;
execed = 1;

error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
&first_exec, ucode, ninst,
&gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
}
if (execed) {
qat_ae_restore_init_lm_gprs(sc, ae, 0,
gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
}

qat_free_mem(ucode);

return error;
}

int
qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc)
{

if (qat_ae_is_active(sc, ae))
return EBUSY;

qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT,
UPC_MASK & upc);
return 0;
}

static inline u_int
qat_aefw_csum_calc(u_int reg, int ch)
{
int i;
u_int topbit = CRC_BITMASK(CRC_WIDTH - 1);
u_int inbyte = (u_int)((reg >> 0x18) ^ ch);

reg ^= inbyte << (CRC_WIDTH - 0x8);
for (i = 0; i < 0x8; i++) {
if (reg & topbit)
reg = (reg << 1) ^ CRC_POLY;
else
reg <<= 1;
}

return (reg & CRC_WIDTHMASK(CRC_WIDTH));
}

u_int
qat_aefw_csum(char *buf, int size)
{
u_int csum = 0;

while (size--) {
csum = qat_aefw_csum_calc(csum, *buf++);
}

return csum;
}

const char *
qat_aefw_uof_string(struct qat_softc *sc, size_t offset)
{
if (offset >= sc->sc_aefw_uof.qafu_str_tab_size)
return NULL;
if (sc->sc_aefw_uof.qafu_str_tab == NULL)
return NULL;

return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset);
}

struct uof_chunk_hdr *
qat_aefw_uof_find_chunk(struct qat_softc *sc,
const char *id, struct uof_chunk_hdr *cur)
{
struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr;
struct uof_chunk_hdr *uch;
int i;

uch = (struct uof_chunk_hdr *)(uoh + 1);
for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) {
if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size)
return NULL;

if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN))
return uch;
}

return NULL;
}

int
qat_aefw_load_mof(struct qat_softc *sc)
{
int error = 0;
firmware_handle_t fh = NULL;
off_t fwsize;

/* load MOF firmware */
error = firmware_open("qat", sc->sc_hw.qhw_mof_fwname, &fh);
if (error) {
aprint_error_dev(sc->sc_dev, "couldn't load mof firmware %s\n",
sc->sc_hw.qhw_mof_fwname);
goto fail;
}

fwsize = firmware_get_size(fh);
if (fwsize == 0 || fwsize > SIZE_MAX) {
error = EINVAL;
goto fail;
}
sc->sc_fw_mof_size = fwsize;
sc->sc_fw_mof = firmware_malloc(sc->sc_fw_mof_size);

error = firmware_read(fh, 0, sc->sc_fw_mof, sc->sc_fw_mof_size);
if (error)
goto fail;

out:
if (fh != NULL)
firmware_close(fh);
return error;
fail:
if (sc->sc_fw_mof != NULL) {
firmware_free(sc->sc_fw_mof, sc->sc_fw_mof_size);
sc->sc_fw_mof = NULL;
}
goto out;
}

int
qat_aefw_load_mmp(struct qat_softc *sc)
{
int error = 0;
firmware_handle_t fh = NULL;
off_t fwsize;

error = firmware_open("qat", sc->sc_hw.qhw_mmp_fwname, &fh);
if (error) {
aprint_error_dev(sc->sc_dev, "couldn't load mmp firmware %s\n",
sc->sc_hw.qhw_mmp_fwname);
goto fail;
}

fwsize = firmware_get_size(fh);
if (fwsize == 0 || fwsize > SIZE_MAX) {
error = EINVAL;
goto fail;
}
sc->sc_fw_mmp_size = fwsize;
sc->sc_fw_mmp = firmware_malloc(sc->sc_fw_mmp_size);

error = firmware_read(fh, 0, sc->sc_fw_mmp, sc->sc_fw_mmp_size);
if (error)
goto fail;

out:
if (fh != NULL)
firmware_close(fh);
return error;
fail:
if (sc->sc_fw_mmp != NULL) {
firmware_free(sc->sc_fw_mmp, sc->sc_fw_mmp_size);
sc->sc_fw_mmp = NULL;
}
goto out;
}

int
qat_aefw_mof_find_uof0(struct qat_softc *sc,
struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head,
u_int nchunk, size_t size, const char *id,
size_t *fwsize, void **fwptr)
{
int i;
char *uof_name;

for (i = 0; i < nchunk; i++) {
struct mof_uof_chunk_hdr *much = &head[i];

if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN))
return EINVAL;

if (much->much_offset + much->much_size > size)
return EINVAL;

if (sc->sc_mof.qmf_sym_size <= much->much_name)
return EINVAL;

uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym +
much->much_name);

if (!strcmp(uof_name, sc->sc_fw_uof_name)) {
*fwptr = (void *)((uintptr_t)muh +
(uintptr_t)much->much_offset);
*fwsize = (size_t)much->much_size;
aprint_verbose_dev(sc->sc_dev,
"%s obj %s at %p size 0x%lx\n",
id, uof_name, *fwptr, *fwsize);
return 0;
}
}

return ENOENT;
}

int
qat_aefw_mof_find_uof(struct qat_softc *sc)
{
struct mof_uof_hdr *uof_hdr, *suof_hdr;
u_int nuof_chunks = 0, nsuof_chunks = 0;
int error;

uof_hdr = sc->sc_mof.qmf_uof_objs;
suof_hdr = sc->sc_mof.qmf_suof_objs;

if (uof_hdr != NULL) {
if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) {
return EINVAL;
}
nuof_chunks = uof_hdr->muh_num_chunks;
}
if (suof_hdr != NULL) {
if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks)
return EINVAL;
nsuof_chunks = suof_hdr->muh_num_chunks;
}

if (nuof_chunks + nsuof_chunks == 0)
return EINVAL;

if (uof_hdr != NULL) {
error = qat_aefw_mof_find_uof0(sc, uof_hdr,
(struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks,
sc->sc_mof.qmf_uof_objs_size, UOF_IMAG,
&sc->sc_fw_uof_size, &sc->sc_fw_uof);
if (error && error != ENOENT)
return error;
}

if (suof_hdr != NULL) {
error = qat_aefw_mof_find_uof0(sc, suof_hdr,
(struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks,
sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG,
&sc->sc_fw_suof_size, &sc->sc_fw_suof);
if (error && error != ENOENT)
return error;
}

if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL)
return ENOENT;

return 0;
}

int
qat_aefw_mof_parse(struct qat_softc *sc)
{
struct mof_file_hdr *mfh;
struct mof_file_chunk_hdr *mfch;
size_t size;
u_int csum;
int error, i;

size = sc->sc_fw_mof_size;

if (size < sizeof(struct mof_file_hdr))
return EINVAL;
size -= sizeof(struct mof_file_hdr);

mfh = sc->sc_fw_mof;

if (mfh->mfh_fid != MOF_FID)
return EINVAL;

csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof +
offsetof(struct mof_file_hdr, mfh_min_ver)),
sc->sc_fw_mof_size -
offsetof(struct mof_file_hdr, mfh_min_ver));
if (mfh->mfh_csum != csum)
return EINVAL;

if (mfh->mfh_min_ver != MOF_MIN_VER ||
mfh->mfh_maj_ver != MOF_MAJ_VER)
return EINVAL;

if (mfh->mfh_max_chunks < mfh->mfh_num_chunks)
return EINVAL;

if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks)
return EINVAL;
mfch = (struct mof_file_chunk_hdr *)(mfh + 1);

for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) {
if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size)
return EINVAL;

if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) {
if (sc->sc_mof.qmf_sym != NULL)
return EINVAL;

sc->sc_mof.qmf_sym =
(void *)((uintptr_t)sc->sc_fw_mof +
(uintptr_t)mfch->mfch_offset + sizeof(u_int));
sc->sc_mof.qmf_sym_size =
*(u_int *)((uintptr_t)sc->sc_fw_mof +
(uintptr_t)mfch->mfch_offset);

if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0)
return EINVAL;
if (mfch->mfch_size != sc->sc_mof.qmf_sym_size +
sizeof(u_int) || mfch->mfch_size == 0)
return EINVAL;
if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym +
sc->sc_mof.qmf_sym_size - 1) != '\0')
return EINVAL;

} else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) {
if (sc->sc_mof.qmf_uof_objs != NULL)
return EINVAL;

sc->sc_mof.qmf_uof_objs =
(void *)((uintptr_t)sc->sc_fw_mof +
(uintptr_t)mfch->mfch_offset);
sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size;

} else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) {
if (sc->sc_mof.qmf_suof_objs != NULL)
return EINVAL;

sc->sc_mof.qmf_suof_objs =
(void *)((uintptr_t)sc->sc_fw_mof +
(uintptr_t)mfch->mfch_offset);
sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size;
}
}

if (sc->sc_mof.qmf_sym == NULL ||
(sc->sc_mof.qmf_uof_objs == NULL &&
sc->sc_mof.qmf_suof_objs == NULL))
return EINVAL;

error = qat_aefw_mof_find_uof(sc);
if (error)
return error;
return 0;
}

int
qat_aefw_uof_parse_image(struct qat_softc *sc,
struct qat_uof_image *qui, struct uof_chunk_hdr *uch)
{
struct uof_image *image;
struct uof_code_page *page;
uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
size_t lim = uch->uch_offset + uch->uch_size, size;
int i, p;

size = uch->uch_size;
if (size < sizeof(struct uof_image))
return EINVAL;
size -= sizeof(struct uof_image);

qui->qui_image = image =
(struct uof_image *)(base + uch->uch_offset);

aprint_verbose_dev(sc->sc_dev,
"uof_image name %s\n",
qat_aefw_uof_string(sc, image->ui_name));
aprint_verbose_dev(sc->sc_dev,
"uof_image ae_assign 0x%08x ctx_assign 0x%08x cpu_type 0x%08x\n",
image->ui_ae_assigned, image->ui_ctx_assigned, image->ui_cpu_type);
aprint_verbose_dev(sc->sc_dev,
"uof_image max_ver 0x%08x min_ver 0x%08x ae_mode 0x%08x\n",
image->ui_max_ver, image->ui_min_ver, image->ui_ae_mode);
aprint_verbose_dev(sc->sc_dev,
"uof_image pages 0x%08x page regions 0x%08x\n",
image->ui_num_pages, image->ui_num_page_regions);

#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \
do { \
u_int nent; \
nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\
if ((lim) < off + sizeof(struct uof_obj_table) + \
sizeof(type) * nent) \
return EINVAL; \
*(np) = nent; \
if (nent > 0) \
*(typep) = (type)((struct uof_obj_table *) \
((base) + (off)) + 1); \
else \
*(typep) = NULL; \
} while (0)

ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg,
struct uof_ae_reg *, base, image->ui_reg_tab, lim);
ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym,
struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim);
ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak,
struct qui_sbreak *, base, image->ui_sbreak_tab, lim);

if (size < sizeof(struct uof_code_page) * image->ui_num_pages)
return EINVAL;
if (__arraycount(qui->qui_pages) < image->ui_num_pages)
return EINVAL;

page = (struct uof_code_page *)(image + 1);

for (p = 0; p < image->ui_num_pages; p++, page++) {
struct qat_uof_page *qup = &qui->qui_pages[p];
struct uof_code_area *uca;

qup->qup_page_num = page->ucp_page_num;
qup->qup_def_page = page->ucp_def_page;
qup->qup_page_region = page->ucp_page_region;
qup->qup_beg_vaddr = page->ucp_beg_vaddr;
qup->qup_beg_paddr = page->ucp_beg_paddr;

ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var,
struct uof_uword_fixup *, base,
page->ucp_uc_var_tab, lim);
ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var,
struct uof_import_var *, base,
page->ucp_imp_var_tab, lim);
ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr,
struct uof_uword_fixup *, base,
page->ucp_imp_expr_tab, lim);
ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg,
struct uof_uword_fixup *, base,
page->ucp_neigh_reg_tab, lim);

if (lim < page->ucp_code_area + sizeof(struct uof_code_area))
return EINVAL;

uca = (struct uof_code_area *)(base + page->ucp_code_area);
qup->qup_num_micro_words = uca->uca_num_micro_words;

ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks,
struct qat_uof_uword_block *, base,
uca->uca_uword_block_tab, lim);

for (i = 0; i < qup->qup_num_uw_blocks; i++) {
u_int uwordoff = ((struct uof_uword_block *)(
&qup->qup_uw_blocks[i]))->uub_uword_offset;

if (lim < uwordoff)
return EINVAL;

qup->qup_uw_blocks[i].quub_micro_words =
(base + uwordoff);
}
}

#undef ASSIGN_OBJ_TAB

return 0;
}

int
qat_aefw_uof_parse_images(struct qat_softc *sc)
{
struct uof_chunk_hdr *uch = NULL;
u_int assigned_ae;
int i, error;

for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) {
uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch);
if (uch == NULL)
break;

if (i >= __arraycount(sc->sc_aefw_uof.qafu_imgs))
return ENOENT;

error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch);
if (error)
return error;

sc->sc_aefw_uof.qafu_num_imgs++;
}

assigned_ae = 0;
for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
assigned_ae |= sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned;
}

return 0;
}

int
qat_aefw_uof_parse(struct qat_softc *sc)
{
struct uof_file_hdr *ufh;
struct uof_file_chunk_hdr *ufch;
struct uof_obj_hdr *uoh;
struct uof_chunk_hdr *uch;
void *uof = NULL;
size_t size, uof_size, hdr_size;
uintptr_t base;
u_int csum;
int i;

size = sc->sc_fw_uof_size;
if (size < MIN_UOF_SIZE)
return EINVAL;
size -= sizeof(struct uof_file_hdr);

ufh = sc->sc_fw_uof;

if (ufh->ufh_id != UOF_FID)
return EINVAL;
if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER)
return EINVAL;

if (ufh->ufh_max_chunks < ufh->ufh_num_chunks)
return EINVAL;
if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks)
return EINVAL;
ufch = (struct uof_file_chunk_hdr *)(ufh + 1);

uof_size = 0;
for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) {
if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size)
return EINVAL;

if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) {
if (uof != NULL)
return EINVAL;

uof =
(void *)((uintptr_t)sc->sc_fw_uof +
ufch->ufch_offset);
uof_size = ufch->ufch_size;

csum = qat_aefw_csum(uof, uof_size);
if (csum != ufch->ufch_csum)
return EINVAL;

aprint_verbose_dev(sc->sc_dev,
"uof at %p size 0x%lx\n",
uof, uof_size);
}
}

if (uof == NULL)
return ENOENT;

size = uof_size;
if (size < sizeof(struct uof_obj_hdr))
return EINVAL;
size -= sizeof(struct uof_obj_hdr);

uoh = uof;

aprint_verbose_dev(sc->sc_dev,
"uof cpu_type 0x%08x min_cpu_ver 0x%04x max_cpu_ver 0x%04x\n",
uoh->uoh_cpu_type, uoh->uoh_min_cpu_ver, uoh->uoh_max_cpu_ver);

if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks)
return EINVAL;

/* Check if the UOF objects are compatible with the chip */
if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0)
return ENOTSUP;

if (uoh->uoh_min_cpu_ver > sc->sc_rev ||
uoh->uoh_max_cpu_ver < sc->sc_rev)
return ENOTSUP;

sc->sc_aefw_uof.qafu_size = uof_size;
sc->sc_aefw_uof.qafu_obj_hdr = uoh;

base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;

/* map uof string-table */
uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL);
if (uch != NULL) {
hdr_size = offsetof(struct uof_str_tab, ust_strings);
sc->sc_aefw_uof.qafu_str_tab =
(void *)(base + uch->uch_offset + hdr_size);
sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size;
}

/* get ustore mem inits table -- should be only one */
uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL);
if (uch != NULL) {
if (uch->uch_size < sizeof(struct uof_obj_table))
return EINVAL;
sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base +
uch->uch_offset))->uot_nentries;
if (sc->sc_aefw_uof.qafu_num_init_mem) {
sc->sc_aefw_uof.qafu_init_mem =
(struct uof_init_mem *)(base + uch->uch_offset +
sizeof(struct uof_obj_table));
sc->sc_aefw_uof.qafu_init_mem_size =
uch->uch_size - sizeof(struct uof_obj_table);
}
}

uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL);
if (uch != NULL) {
if (uch->uch_size < sizeof(struct uof_obj_table) +
sizeof(struct uof_var_mem_seg))
return EINVAL;
sc->sc_aefw_uof.qafu_var_mem_seg =
(struct uof_var_mem_seg *)(base + uch->uch_offset +
sizeof(struct uof_obj_table));
}

return qat_aefw_uof_parse_images(sc);
}

int
qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi,
struct suof_chunk_hdr *sch)
{
struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
struct simg_ae_mode *ae_mode;
u_int maj_ver;

qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset +
sizeof(struct suof_obj_hdr);
qsi->qsi_simg_len =
((struct suof_obj_hdr *)
(qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length;

qsi->qsi_css_header = qsi->qsi_simg_buf;
qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr);
qsi->qsi_css_signature = qsi->qsi_css_key +
CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN;

ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg;
qsi->qsi_ae_mask = ae_mode->sam_ae_mask;
qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name;
qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data;
qsi->qsi_fw_type = ae_mode->sam_fw_type;

if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type)
return EINVAL;

maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff;
if ((maj_ver > ae_mode->sam_devmax_ver) ||
(maj_ver < ae_mode->sam_devmin_ver)) {
return EINVAL;
}

return 0;
}

int
qat_aefw_suof_parse(struct qat_softc *sc)
{
struct suof_file_hdr *sfh;
struct suof_chunk_hdr *sch;
struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
struct qat_suof_image *qsi;
size_t size;
u_int csum;
int ae0_img = MAX_AE;
int i, error;

size = sc->sc_fw_suof_size;
if (size < sizeof(struct suof_file_hdr))
return EINVAL;

sfh = sc->sc_fw_suof;

if (sfh->sfh_file_id != SUOF_FID)
return EINVAL;
if (sfh->sfh_fw_type != 0)
return EINVAL;
if (sfh->sfh_num_chunks <= 1)
return EINVAL;
if (sfh->sfh_min_ver != SUOF_MIN_VER ||
sfh->sfh_maj_ver != SUOF_MAJ_VER)
return EINVAL;

csum = qat_aefw_csum((char *)&sfh->sfh_min_ver,
size - offsetof(struct suof_file_hdr, sfh_min_ver));
if (csum != sfh->sfh_check_sum)
return EINVAL;

size -= sizeof(struct suof_file_hdr);

qafs->qafs_file_id = SUOF_FID;
qafs->qafs_suof_buf = sc->sc_fw_suof;
qafs->qafs_suof_size = sc->sc_fw_suof_size;
qafs->qafs_check_sum = sfh->sfh_check_sum;
qafs->qafs_min_ver = sfh->sfh_min_ver;
qafs->qafs_maj_ver = sfh->sfh_maj_ver;
qafs->qafs_fw_type = sfh->sfh_fw_type;

if (size < sizeof(struct suof_chunk_hdr))
return EINVAL;
sch = (struct suof_chunk_hdr *)(sfh + 1);
size -= sizeof(struct suof_chunk_hdr);

if (size < sizeof(struct suof_str_tab))
return EINVAL;
size -= offsetof(struct suof_str_tab, sst_strings);

qafs->qafs_sym_size = ((struct suof_str_tab *)
(qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length;
if (size < qafs->qafs_sym_size)
return EINVAL;
qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset +
offsetof(struct suof_str_tab, sst_strings);

qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1;
if (qafs->qafs_num_simgs == 0)
return EINVAL;

qsi = qat_alloc_mem(
sizeof(struct qat_suof_image) * qafs->qafs_num_simgs);
qafs->qafs_simg = qsi;

for (i = 0; i < qafs->qafs_num_simgs; i++) {
error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]);
if (error)
return error;
if ((qsi[i].qsi_ae_mask & 0x1) != 0)
ae0_img = i;
}

if (ae0_img != qafs->qafs_num_simgs - 1) {
struct qat_suof_image last_qsi;

memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1],
sizeof(struct qat_suof_image));
memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img],
sizeof(struct qat_suof_image));
memcpy(&qsi[ae0_img], &last_qsi,
sizeof(struct qat_suof_image));
}

return 0;
}

int
qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size,
struct qat_dmamem *dma)
{
struct css_hdr *css = (struct css_hdr *)image;
struct auth_chunk *auth_chunk;
struct fw_auth_desc *auth_desc;
size_t mapsize, simg_offset = sizeof(struct auth_chunk);
bus_size_t bus_addr;
uintptr_t virt_addr;
int error;

if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN)
return EINVAL;

mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ?
CSS_AE_SIMG_LEN + simg_offset :
size + CSS_FWSK_PAD_LEN + simg_offset;
error = qat_alloc_dmamem(sc, dma, mapsize, PAGE_SIZE);
if (error)
return error;

memset(dma->qdm_dma_vaddr, 0, mapsize);

auth_chunk = dma->qdm_dma_vaddr;
auth_chunk->ac_chunk_size = mapsize;
auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_map->dm_segs[0].ds_addr;

virt_addr = (uintptr_t)dma->qdm_dma_vaddr;
virt_addr += simg_offset;
bus_addr = auth_chunk->ac_chunk_bus_addr;
bus_addr += simg_offset;

auth_desc = &auth_chunk->ac_fw_auth_desc;
auth_desc->fad_css_hdr_high = bus_addr >> 32;
auth_desc->fad_css_hdr_low = bus_addr;

memcpy((void *)virt_addr, image, sizeof(struct css_hdr));
/* pub key */
virt_addr += sizeof(struct css_hdr);
bus_addr += sizeof(struct css_hdr);
image += sizeof(struct css_hdr);

auth_desc->fad_fwsk_pub_high = bus_addr >> 32;
auth_desc->fad_fwsk_pub_low = bus_addr;

memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN);
memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN);
memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN),
image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t));

virt_addr += CSS_FWSK_PUB_LEN;
bus_addr += CSS_FWSK_PUB_LEN;
image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;

auth_desc->fad_signature_high = bus_addr >> 32;
auth_desc->fad_signature_low = bus_addr;

memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN);
#ifdef QAT_DUMP
qat_dump_raw(QAT_DUMP_AEFW, "aefw signature", image, CSS_SIGNATURE_LEN);
#endif

virt_addr += CSS_SIGNATURE_LEN;
bus_addr += CSS_SIGNATURE_LEN;
image += CSS_SIGNATURE_LEN;

auth_desc->fad_img_high = bus_addr >> 32;
auth_desc->fad_img_low = bus_addr;
auth_desc->fad_img_len = size - AE_IMG_OFFSET;

memcpy((void *)virt_addr, image, auth_desc->fad_img_len);

if (css->css_fw_type == CSS_AE_FIRMWARE) {
auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high;
auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low;

bus_addr += sizeof(struct simg_ae_mode);

auth_desc->fad_img_ae_init_data_high = bus_addr >> 32;
auth_desc->fad_img_ae_init_data_low = bus_addr;

bus_addr += SIMG_AE_INIT_SEQ_LEN;

auth_desc->fad_img_ae_insts_high = bus_addr >> 32;
auth_desc->fad_img_ae_insts_low = bus_addr;
} else {
auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high;
auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low;
}

bus_dmamap_sync(sc->sc_dmat, dma->qdm_dma_map, 0,
dma->qdm_dma_map->dm_mapsize,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

return 0;
}

int
qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma)
{
bus_addr_t addr = dma->qdm_dma_map->dm_segs[0].ds_addr;
uint32_t fcu, sts;
int retry = 0;

qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, addr >> 32);
qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr);
qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH);

do {
delay(FW_AUTH_WAIT_PERIOD * 1000);
fcu = qat_cap_global_read_4(sc, FCU_STATUS);
sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
if (sts == FCU_STATUS_STS_VERI_FAIL)
goto fail;
if (fcu & FCU_STATUS_AUTHFWLD &&
sts == FCU_STATUS_STS_VERI_DONE) {
return 0;
}
} while (retry++ < FW_AUTH_MAX_RETRY);

fail:
aprint_error_dev(sc->sc_dev,
"firmware authentication error: status 0x%08x retry %d\n",
fcu, retry);
return EINVAL;
}

int
qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma)
{
struct simg_ae_mode *ae_mode;
uint32_t fcu, sts, loaded;
u_int mask;
u_char ae;
int retry = 0;

ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr +
sizeof(struct auth_chunk) + sizeof(struct css_hdr) +
CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN);

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
if (!(mask & 1))
continue;
if (!((ae_mode->sam_ae_mask >> ae) & 0x1))
continue;
if (qat_ae_is_active(sc, ae)) {
aprint_error_dev(sc->sc_dev, "AE %d is active\n", ae);
return EINVAL;
}
qat_cap_global_write_4(sc, FCU_CTRL,
FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE));
do {
delay(FW_AUTH_WAIT_PERIOD * 1000);
fcu = qat_cap_global_read_4(sc, FCU_STATUS);
sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE);
if (sts == FCU_STATUS_STS_LOAD_DONE &&
(loaded & (1 << ae))) {
break;
}
} while (retry++ < FW_AUTH_MAX_RETRY);

if (retry > FW_AUTH_MAX_RETRY) {
aprint_error_dev(sc->sc_dev,
"firmware load timeout: status %08x\n", fcu);
return EINVAL;
}
}

return 0;
}

int
qat_aefw_suof_write(struct qat_softc *sc)
{
struct qat_suof_image *qsi = NULL;
int i, error = 0;

for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) {
qsi = &sc->sc_aefw_suof.qafs_simg[i];
error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf,
qsi->qsi_simg_len, &qsi->qsi_dma);
if (error)
return error;
error = qat_aefw_auth(sc, &qsi->qsi_dma);
if (error)
goto fail;
error = qat_aefw_suof_load(sc, &qsi->qsi_dma);
if (error)
goto fail;

qat_free_dmamem(sc, &qsi->qsi_dma);
}

return 0;
fail:
if (qsi != NULL)
qat_free_dmamem(sc, &qsi->qsi_dma);
return error;
}

int
qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae,
struct qat_uof_image *qui)
{
struct qat_ae_slice *slice;
int i, npages, nregions;

if (qae->qae_num_slices >= __arraycount(qae->qae_slices))
return ENOENT;

if (qui->qui_image->ui_ae_mode &
(AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) {
/* XXX */
aprint_error_dev(sc->sc_dev,
"shared ae mode is not supported yet\n");
return ENOTSUP;
}

qae->qae_shareable_ustore = 0; /* XXX */
qae->qae_effect_ustore_size = USTORE_SIZE;

slice = &qae->qae_slices[qae->qae_num_slices];

slice->qas_image = qui;
slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned;

nregions = qui->qui_image->ui_num_page_regions;
npages = qui->qui_image->ui_num_pages;

if (nregions > __arraycount(slice->qas_regions))
return ENOENT;
if (npages > __arraycount(slice->qas_pages))
return ENOENT;

for (i = 0; i < nregions; i++) {
SIMPLEQ_INIT(&slice->qas_regions[i].qar_waiting_pages);
}
for (i = 0; i < npages; i++) {
struct qat_ae_page *page = &slice->qas_pages[i];
int region;

page->qap_page = &qui->qui_pages[i];
region = page->qap_page->qup_page_region;
if (region >= nregions)
return EINVAL;

page->qap_region = &slice->qas_regions[region];
aprint_verbose_dev(sc->sc_dev,
"ae %p slice %d page %d assign region %d\n",
qae, qae->qae_num_slices, i, region);
}

qae->qae_num_slices++;

return 0;
}

int
qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae)
{
struct uof_image *image;
struct qat_ae *qae = &(QAT_AE(sc, ae));
int s;
u_char nn_mode;

for (s = 0; s < qae->qae_num_slices; s++) {
if (qae->qae_slices[s].qas_image == NULL)
continue;

image = qae->qae_slices[s].qas_image->qui_image;
qat_ae_write_ctx_mode(sc, ae,
__SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE));

nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE);
if (nn_mode != AE_MODE_NN_MODE_DONTCARE)
qat_ae_write_nn_mode(sc, ae, nn_mode);

qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0,
__SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0));
qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1,
__SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1));

qat_ae_write_shared_cs_mode(sc, ae,
__SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE));
qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size,
__SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED),
qae->qae_reloc_ustore_dram);
}

return 0;
}

int
qat_aefw_uof_init(struct qat_softc *sc)
{
int ae, i, error;
uint32_t mask;

for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
struct qat_ae *qae;

if (!(mask & 1))
continue;

qae = &(QAT_AE(sc, ae));

for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned &
(1 << ae)) == 0)
continue;

error = qat_aefw_uof_assign_image(sc, qae,
&sc->sc_aefw_uof.qafu_imgs[i]);
if (error)
return error;
}

/* XXX UcLo_initNumUwordUsed */

qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */

error = qat_aefw_uof_init_ae(sc, ae);
if (error)
return error;
}

return 0;
}

int
qat_aefw_load(struct qat_softc *sc)
{
int error;

error = qat_aefw_load_mof(sc);
if (error)
return error;

error = qat_aefw_load_mmp(sc);
if (error)
return error;

error = qat_aefw_mof_parse(sc);
if (error) {
aprint_error_dev(sc->sc_dev, "couldn't parse mof: %d\n", error);
return error;
}

if (sc->sc_hw.qhw_fw_auth) {
error = qat_aefw_suof_parse(sc);
if (error) {
aprint_error_dev(sc->sc_dev, "couldn't parse suof: %d\n",
error);
return error;
}

error = qat_aefw_suof_write(sc);
if (error) {
aprint_error_dev(sc->sc_dev,
"could not write firmware: %d\n", error);
return error;
}

} else {
error = qat_aefw_uof_parse(sc);
if (error) {
aprint_error_dev(sc->sc_dev, "couldn't parse uof: %d\n",
error);
return error;
}

error = qat_aefw_uof_init(sc);
if (error) {
aprint_error_dev(sc->sc_dev,
"couldn't init for aefw: %d\n", error);
return error;
}

error = qat_aefw_uof_write(sc);
if (error) {
aprint_error_dev(sc->sc_dev,
"Could not write firmware: %d\n", error);
return error;
}
}

return 0;
}

int
qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
uint32_t fcu;
int retry = 0;

if (sc->sc_hw.qhw_fw_auth) {
qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START);
do {
delay(FW_AUTH_WAIT_PERIOD * 1000);
fcu = qat_cap_global_read_4(sc, FCU_STATUS);
if (fcu & FCU_STATUS_DONE)
return 0;
} while (retry++ < FW_AUTH_MAX_RETRY);

aprint_error_dev(sc->sc_dev,
"firmware start timeout: status %08x\n", fcu);
return EINVAL;
} else {
qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX,
CTX_WAKEUP_EVENTS_INDIRECT,
CTX_WAKEUP_EVENTS_INDIRECT_SLEEP);
qat_ae_enable_ctx(sc, ae, ctx_mask);
}

return 0;
}

int
qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim)
{
struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
struct qat_ae_batch_init_list *qabi_list;
struct uof_mem_val_attr *memattr;
size_t *curinit;
u_long ael;
int i;
const char *sym;
char *ep;

memattr = (struct uof_mem_val_attr *)(uim + 1);

switch (uim->uim_region) {
case LMEM_REGION:
if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) {
aprint_error_dev(sc->sc_dev,
"Invalid lmem addr or bytes\n");
return ENOBUFS;
}
if (uim->uim_scope != UOF_SCOPE_LOCAL)
return EINVAL;
sym = qat_aefw_uof_string(sc, uim->uim_sym_name);
ael = strtoul(sym, &ep, 10);
if (ep == sym || ael > MAX_AE)
return EINVAL;
if ((sc->sc_ae_mask & (1 << ael)) == 0)
return 0; /* ae is fused out */

curinit = &qafu->qafu_num_lm_init[ael];
qabi_list = &qafu->qafu_lm_init[ael];

for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) {
struct qat_ae_batch_init *qabi;

qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init));
if (*curinit == 0)
SIMPLEQ_INIT(qabi_list);
SIMPLEQ_INSERT_TAIL(qabi_list, qabi, qabi_next);

qabi->qabi_ae = (u_int)ael;
qabi->qabi_addr =
uim->uim_addr + memattr->umva_byte_offset;
qabi->qabi_value = &memattr->umva_value;
qabi->qabi_size = 4;
qafu->qafu_num_lm_init_inst[ael] +=
qat_ae_get_inst_num(qabi->qabi_size);
(*curinit)++;
if (*curinit >= MAX_LMEM_REG) {
aprint_error_dev(sc->sc_dev,
"Invalid lmem val attr\n");
return ENOBUFS;
}
}
break;
case SRAM_REGION:
case DRAM_REGION:
case DRAM1_REGION:
case SCRATCH_REGION:
case UMEM_REGION:
/* XXX */
/* fallthrough */
default:
aprint_error_dev(sc->sc_dev,
"unsupported memory region to init: %d\n",
uim->uim_region);
return ENOTSUP;
}

return 0;
}

void
qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae)
{
struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
struct qat_ae_batch_init *qabi;

while ((qabi = SIMPLEQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) {
SIMPLEQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next);
qat_free_mem(qabi);
}

qafu->qafu_num_lm_init[ae] = 0;
qafu->qafu_num_lm_init_inst[ae] = 0;
}

int
qat_aefw_init_ustore(struct qat_softc *sc)
{
uint64_t *fill;
uint32_t dont_init;
int a, i, p;
int error = 0;
int usz, end, start;
u_char ae, nae;

fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t));

for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) {
struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a];
struct uof_image *ui = qui->qui_image;

for (i = 0; i < MAX_USTORE; i++)
memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t));
/*
* Compute do_not_init value as a value that will not be equal
* to fill data when cast to an int
*/
dont_init = 0;
if (dont_init == (uint32_t)fill[0])
dont_init = 0xffffffff;

for (p = 0; p < ui->ui_num_pages; p++) {
struct qat_uof_page *qup = &qui->qui_pages[p];
if (!qup->qup_def_page)
continue;

for (i = qup->qup_beg_paddr;
i < qup->qup_beg_paddr + qup->qup_num_micro_words;
i++ ) {
fill[i] = (uint64_t)dont_init;
}
}

for (ae = 0; ae < sc->sc_ae_num; ae++) {
KASSERT(ae < UOF_MAX_NUM_OF_AE);
if ((ui->ui_ae_assigned & (1 << ae)) == 0)
continue;

if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) {
qat_ae_get_shared_ustore_ae(ae, &nae);
if (ui->ui_ae_assigned & (1 << ae))
continue;
}
usz = QAT_AE(sc, ae).qae_effect_ustore_size;

/* initialize the areas not going to be overwritten */
end = -1;
do {
/* find next uword that needs to be initialized */
for (start = end + 1; start < usz; start++) {
if ((uint32_t)fill[start] != dont_init)
break;
}
/* see if there are no more such uwords */
if (start >= usz)
break;
for (end = start + 1; end < usz; end++) {
if ((uint32_t)fill[end] == dont_init)
break;
}
if (QAT_AE(sc, ae).qae_shareable_ustore) {
error = ENOTSUP; /* XXX */
goto out;
} else {
error = qat_ae_ucode_write(sc, ae,
start, end - start, &fill[start]);
if (error) {
goto out;
}
}

} while (end < usz);
}
}

out:
qat_free_mem(fill);
return error;
}

int
qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask,
enum aereg_type regtype, u_short regaddr, u_int value)
{
int error = 0;
u_char ctx;

switch (regtype) {
case AEREG_GPA_REL:
case AEREG_GPB_REL:
case AEREG_SR_REL:
case AEREG_SR_RD_REL:
case AEREG_SR_WR_REL:
case AEREG_DR_REL:
case AEREG_DR_RD_REL:
case AEREG_DR_WR_REL:
case AEREG_NEIGH_REL:
/* init for all valid ctx */
for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
if ((ctx_mask & (1 << ctx)) == 0)
continue;
error = qat_aereg_rel_data_write(sc, ae, ctx, regtype,
regaddr, value);
}
break;
case AEREG_GPA_ABS:
case AEREG_GPB_ABS:
case AEREG_SR_ABS:
case AEREG_SR_RD_ABS:
case AEREG_SR_WR_ABS:
case AEREG_DR_ABS:
case AEREG_DR_RD_ABS:
case AEREG_DR_WR_ABS:
error = qat_aereg_abs_data_write(sc, ae, regtype,
regaddr, value);
break;
default:
error = EINVAL;
break;
}

return error;
}

int
qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae,
struct qat_uof_image *qui)
{
u_int i, expres;
u_char ctx_mask;

for (i = 0; i < qui->qui_num_init_reg_sym; i++) {
struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i];

if (uirs->uirs_value_type == EXPR_VAL) {
/* XXX */
aprint_error_dev(sc->sc_dev,
"does not support initializing EXPR_VAL\n");
return ENOTSUP;
} else {
expres = uirs->uirs_value;
}

switch (uirs->uirs_init_type) {
case INIT_REG:
if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
AE_MODE_CTX_MODE) == MAX_AE_CTX) {
ctx_mask = 0xff; /* 8-ctx mode */
} else {
ctx_mask = 0x55; /* 4-ctx mode */
}
qat_aefw_init_reg(sc, ae, ctx_mask,
(enum aereg_type)uirs->uirs_reg_type,
(u_short)uirs->uirs_addr_offset, expres);
break;
case INIT_REG_CTX:
if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
AE_MODE_CTX_MODE) == MAX_AE_CTX) {
ctx_mask = 0xff; /* 8-ctx mode */
} else {
ctx_mask = 0x55; /* 4-ctx mode */
}
if (((1 << uirs->uirs_ctx) & ctx_mask) == 0)
return EINVAL;
qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx,
(enum aereg_type)uirs->uirs_reg_type,
(u_short)uirs->uirs_addr_offset, expres);
break;
case INIT_EXPR:
case INIT_EXPR_ENDIAN_SWAP:
default:
aprint_error_dev(sc->sc_dev,
"does not support initializing init_type %d\n",
uirs->uirs_init_type);
return ENOTSUP;
}
}

return 0;
}

int
qat_aefw_init_memory(struct qat_softc *sc)
{
struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
size_t uimsz, initmemsz = qafu->qafu_init_mem_size;
struct uof_init_mem *uim;
int error, i;
u_char ae;

uim = qafu->qafu_init_mem;
for (i = 0; i < qafu->qafu_num_init_mem; i++) {
uimsz = sizeof(struct uof_init_mem) +
sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr;
if (uimsz > initmemsz) {
aprint_error_dev(sc->sc_dev,
"invalid uof_init_mem or uof_mem_val_attr size\n");
return EINVAL;
}

if (uim->uim_num_bytes > 0) {
error = qat_aefw_init_memory_one(sc, uim);
if (error) {
aprint_error_dev(sc->sc_dev,
"Could not init ae memory: %d\n", error);
return error;
}
}
uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz);
initmemsz -= uimsz;
}

/* run Batch put LM API */
for (ae = 0; ae < MAX_AE; ae++) {
error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae],
qafu->qafu_num_lm_init_inst[ae]);
if (error)
aprint_error_dev(sc->sc_dev, "Could not put lm\n");

qat_aefw_free_lm_init(sc, ae);
}

error = qat_aefw_init_ustore(sc);

/* XXX run Batch put LM API */

return error;
}

int
qat_aefw_init_globals(struct qat_softc *sc)
{
struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
int error, i, p, s;
u_char ae;

/* initialize the memory segments */
if (qafu->qafu_num_init_mem > 0) {
error = qat_aefw_init_memory(sc);
if (error)
return error;
} else {
error = qat_aefw_init_ustore(sc);
if (error)
return error;
}

/* XXX bind import variables with ivd values */

/* XXX bind the uC global variables
* local variables will done on-the-fly */
for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) {
struct qat_uof_page *qup =
&sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p];
if (qup->qup_num_uw_blocks &&
(qup->qup_num_uc_var || qup->qup_num_imp_var)) {
aprint_error_dev(sc->sc_dev,
"not support uC global variables\n");
return ENOTSUP;
}
}
}

for (ae = 0; ae < sc->sc_ae_num; ae++) {
struct qat_ae *qae = &(QAT_AE(sc, ae));

for (s = 0; s < qae->qae_num_slices; s++) {
struct qat_ae_slice *qas = &qae->qae_slices[s];

if (qas->qas_image == NULL)
continue;

error =
qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image);
if (error)
return error;
}
}

return 0;
}

uint64_t
qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup,
u_int addr)
{
uint64_t uinst = 0;
u_int i;

/* find the block */
for (i = 0; i < qup->qup_num_uw_blocks; i++) {
struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i];

if ((addr >= quub->quub_start_addr) &&
(addr <= (quub->quub_start_addr +
(quub->quub_num_words - 1)))) {
/* unpack n bytes and assigned to the 64-bit uword value.
note: the microwords are stored as packed bytes.
*/
addr -= quub->quub_start_addr;
addr *= AEV2_PACKED_UWORD_BYTES;
memcpy(&uinst,
(void *)((uintptr_t)quub->quub_micro_words + addr),
AEV2_PACKED_UWORD_BYTES);
uinst = uinst & UWORD_MASK;

return uinst;
}
}

return INVLD_UWORD;
}

int
qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup)
{
struct qat_ae *qae = &(QAT_AE(sc, ae));
uint64_t fill, *ucode_cpybuf;
u_int error, i, upaddr, uraddr, ninst, cpylen;

if (qup->qup_num_uc_var || qup->qup_num_neigh_reg ||
qup->qup_num_imp_var || qup->qup_num_imp_expr) {
aprint_error_dev(sc->sc_dev,
"does not support fixup locals\n");
return ENOTSUP;
}

ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t));

/* XXX get fill-pattern from an image -- they are all the same */
memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern,
sizeof(uint64_t));

upaddr = qup->qup_beg_paddr;
uraddr = 0;
ninst = qup->qup_num_micro_words;
while (ninst > 0) {
cpylen = uimin(ninst, UWORD_CPYBUF_SIZE);

/* load the buffer */
for (i = 0; i < cpylen; i++) {
/* keep below code structure in case there are
* different handling for shared secnarios */
if (!qae->qae_shareable_ustore) {
/* qat_aefw_get_uof_inst() takes an address that
* is relative to the start of the page.
* So we don't need to add in the physical
* offset of the page. */
if (qup->qup_page_region != 0) {
/* XXX */
aprint_error_dev(sc->sc_dev,
"region != 0 is not supported\n");
qat_free_mem(ucode_cpybuf);
return ENOTSUP;
} else {
/* for mixing case, it should take
* physical address */
ucode_cpybuf[i] = qat_aefw_get_uof_inst(
sc, qup, upaddr + i);
if (ucode_cpybuf[i] == INVLD_UWORD) {
/* fill hole in the uof */
ucode_cpybuf[i] = fill;
}
}
} else {
/* XXX */
qat_free_mem(ucode_cpybuf);
return ENOTSUP;
}
}

/* copy the buffer to ustore */
if (!qae->qae_shareable_ustore) {
error = qat_ae_ucode_write(sc, ae, upaddr, cpylen,
ucode_cpybuf);
if (error)
return error;
} else {
/* XXX */
qat_free_mem(ucode_cpybuf);
return ENOTSUP;
}
upaddr += cpylen;
uraddr += cpylen;
ninst -= cpylen;
}

qat_free_mem(ucode_cpybuf);

return 0;
}

int
qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui)
{
struct uof_image *ui = qui->qui_image;
struct qat_ae_page *qap;
u_int s, p, c;
int error;
u_char ae, ctx_mask;

aprint_verbose_dev(sc->sc_dev,
"aefw writing uof %s\n",
qat_aefw_uof_string(sc, qui->qui_image->ui_name));

error = qat_aefw_init_globals(sc);
if (error) {
aprint_error_dev(sc->sc_dev,
"Could not initialize globals\n");
return error;
}

if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX)
ctx_mask = 0xff; /* 8-ctx mode */
else
ctx_mask = 0x55; /* 4-ctx mode */

/* load the default page and set assigned CTX PC
* to the entrypoint address */
for (ae = 0; ae < sc->sc_ae_num; ae++) {
struct qat_ae *qae = &(QAT_AE(sc, ae));
struct qat_ae_slice *qas;
u_int metadata;

KASSERT(ae < UOF_MAX_NUM_OF_AE);

if ((ui->ui_ae_assigned & (1 << ae)) == 0)
continue;

/* find the slice to which this image is assigned */
for (s = 0; s < qae->qae_num_slices; s++) {
qas = &qae->qae_slices[s];
if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask)
break;
}
if (s >= qae->qae_num_slices)
continue;

qas = &qae->qae_slices[s];

for (p = 0; p < ui->ui_num_pages; p++) {
qap = &qas->qas_pages[p];

/* Only load pages loaded by default */
if (!qap->qap_page->qup_def_page)
continue;

error = qat_aefw_do_pagein(sc, ae, qap->qap_page);
if (error)
return error;
}

metadata = qas->qas_image->qui_image->ui_app_metadata;
if (metadata != 0xffffffff) {
aprint_normal_dev(sc->sc_dev,
"loaded firmware: %s\n",
qat_aefw_uof_string(sc, metadata));
}

/* Assume starting page is page 0 */
qap = &qas->qas_pages[0];
for (c = 0; c < MAX_AE_CTX; c++) {
if (ctx_mask & (1 << c))
qas->qas_cur_pages[c] = qap;
else
qas->qas_cur_pages[c] = NULL;
}

/* set the live context */
qae->qae_live_ctx_mask = ui->ui_ctx_assigned;

/* set context PC to the image entrypoint address */
error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned,
ui->ui_entry_address);
if (error)
return error;
}

/* XXX store the checksum for convenience */

return 0;
}

int
qat_aefw_uof_write(struct qat_softc *sc)
{
int error = 0;
int i;

for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
error = qat_aefw_uof_write_one(sc,
&sc->sc_aefw_uof.qafu_imgs[i]);
if (error)
break;
}

/* XXX UcLo_computeFreeUstore */

return error;
}