/*      $NetBSD: mly.c,v 1.56 2021/09/03 22:33:17 andvar Exp $  */

/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Andrew Doran, Thor Lancelot Simon, and Eric Haszlakiewicz.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*-
* Copyright (c) 2000, 2001 Michael Smith
* Copyright (c) 2000 BSDi
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from FreeBSD: mly.c,v 1.8 2001/07/14 00:12:22 msmith Exp
*/

/*
* Driver for the Mylex AcceleRAID and eXtremeRAID family with v6 firmware.
*
* TODO:
*
* o Make mly->mly_btl a hash, then MLY_BTL_RESCAN becomes a SIMPLEQ.
* o Handle FC and multiple LUNs.
* o Fix mmbox usage.
* o Fix transfer speed fudge.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: mly.c,v 1.56 2021/09/03 22:33:17 andvar Exp $");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/buf.h>
#include <sys/endian.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/ioctl.h>
#include <sys/scsiio.h>
#include <sys/kthread.h>
#include <sys/kauth.h>

#include <sys/bus.h>

#include <dev/scsipi/scsi_all.h>
#include <dev/scsipi/scsipi_all.h>
#include <dev/scsipi/scsiconf.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#include <dev/pci/mlyreg.h>
#include <dev/pci/mlyio.h>
#include <dev/pci/mlyvar.h>
#include <dev/pci/mly_tables.h>

static void     mly_attach(device_t, device_t, void *);
static int      mly_match(device_t, cfdata_t, void *);
static const    struct mly_ident *mly_find_ident(struct pci_attach_args *);
static int      mly_fwhandshake(struct mly_softc *);
static int      mly_flush(struct mly_softc *);
static int      mly_intr(void *);
static void     mly_shutdown(void *);

static int      mly_alloc_ccbs(struct mly_softc *);
static void     mly_check_event(struct mly_softc *);
static void     mly_complete_event(struct mly_softc *, struct mly_ccb *);
static void     mly_complete_rescan(struct mly_softc *, struct mly_ccb *);
static int      mly_dmamem_alloc(struct mly_softc *, int, bus_dmamap_t *,
                                void **, bus_addr_t *, bus_dma_segment_t *);
static void     mly_dmamem_free(struct mly_softc *, int, bus_dmamap_t,
                               void *, bus_dma_segment_t *);
static int      mly_enable_mmbox(struct mly_softc *);
static void     mly_fetch_event(struct mly_softc *);
static int      mly_get_controllerinfo(struct mly_softc *);
static int      mly_get_eventstatus(struct mly_softc *);
static int      mly_ioctl(struct mly_softc *, struct mly_cmd_ioctl *,
                         void **, size_t, void *, size_t *);
static void     mly_padstr(char *, const char *, int);
static void     mly_process_event(struct mly_softc *, struct mly_event *);
static void     mly_release_ccbs(struct mly_softc *);
static int      mly_scan_btl(struct mly_softc *, int, int);
static void     mly_scan_channel(struct mly_softc *, int);
static void     mly_thread(void *);

static int      mly_ccb_alloc(struct mly_softc *, struct mly_ccb **);
static void     mly_ccb_complete(struct mly_softc *, struct mly_ccb *);
static void     mly_ccb_enqueue(struct mly_softc *, struct mly_ccb *);
static void     mly_ccb_free(struct mly_softc *, struct mly_ccb *);
static int      mly_ccb_map(struct mly_softc *, struct mly_ccb *);
static int      mly_ccb_poll(struct mly_softc *, struct mly_ccb *, int);
static int      mly_ccb_submit(struct mly_softc *, struct mly_ccb *);
static void     mly_ccb_unmap(struct mly_softc *, struct mly_ccb *);
static int      mly_ccb_wait(struct mly_softc *, struct mly_ccb *, int);

static void     mly_get_xfer_mode(struct mly_softc *, int,
                                 struct scsipi_xfer_mode *);
static void     mly_scsipi_complete(struct mly_softc *, struct mly_ccb *);
static int      mly_scsipi_ioctl(struct scsipi_channel *, u_long, void *,
                                int, struct proc *);
static void     mly_scsipi_minphys(struct buf *);
static void     mly_scsipi_request(struct scsipi_channel *,
                                  scsipi_adapter_req_t, void *);

static int      mly_user_command(struct mly_softc *, struct mly_user_command *);
static int      mly_user_health(struct mly_softc *, struct mly_user_health *);

extern struct   cfdriver mly_cd;

CFATTACH_DECL_NEW(mly, sizeof(struct mly_softc),
   mly_match, mly_attach, NULL, NULL);

dev_type_open(mlyopen);
dev_type_close(mlyclose);
dev_type_ioctl(mlyioctl);

const struct cdevsw mly_cdevsw = {
       .d_open = mlyopen,
       .d_close = mlyclose,
       .d_read = noread,
       .d_write = nowrite,
       .d_ioctl = mlyioctl,
       .d_stop = nostop,
       .d_tty = notty,
       .d_poll = nopoll,
       .d_mmap = nommap,
       .d_kqfilter = nokqfilter,
       .d_discard = nodiscard,
       .d_flag = D_OTHER
};

static struct mly_ident {
       u_short vendor;
       u_short product;
       u_short subvendor;
       u_short subproduct;
       int     hwif;
       const char      *desc;
} const mly_ident[] = {
       {
               PCI_VENDOR_MYLEX,
               PCI_PRODUCT_MYLEX_EXTREMERAID,
               PCI_VENDOR_MYLEX,
               0x0040,
               MLY_HWIF_STRONGARM,
               "eXtremeRAID 2000"
       },
       {
               PCI_VENDOR_MYLEX,
               PCI_PRODUCT_MYLEX_EXTREMERAID,
               PCI_VENDOR_MYLEX,
               0x0030,
               MLY_HWIF_STRONGARM,
               "eXtremeRAID 3000"
       },
       {
               PCI_VENDOR_MYLEX,
               PCI_PRODUCT_MYLEX_ACCELERAID,
               PCI_VENDOR_MYLEX,
               0x0050,
               MLY_HWIF_I960RX,
               "AcceleRAID 352"
       },
       {
               PCI_VENDOR_MYLEX,
               PCI_PRODUCT_MYLEX_ACCELERAID,
               PCI_VENDOR_MYLEX,
               0x0052,
               MLY_HWIF_I960RX,
               "AcceleRAID 170"
       },
       {
               PCI_VENDOR_MYLEX,
               PCI_PRODUCT_MYLEX_ACCELERAID,
               PCI_VENDOR_MYLEX,
               0x0054,
               MLY_HWIF_I960RX,
               "AcceleRAID 160"
       },
};

static void     *mly_sdh;

/*
* Try to find a `mly_ident' entry corresponding to this board.
*/
static const struct mly_ident *
mly_find_ident(struct pci_attach_args *pa)
{
       const struct mly_ident *mpi, *maxmpi;
       pcireg_t reg;

       mpi = mly_ident;
       maxmpi = mpi + sizeof(mly_ident) / sizeof(mly_ident[0]);

       if (PCI_CLASS(pa->pa_class) == PCI_CLASS_I2O)
               return (NULL);

       for (; mpi < maxmpi; mpi++) {
               if (PCI_VENDOR(pa->pa_id) != mpi->vendor ||
                   PCI_PRODUCT(pa->pa_id) != mpi->product)
                       continue;

               if (mpi->subvendor == 0x0000)
                       return (mpi);

               reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);

               if (PCI_VENDOR(reg) == mpi->subvendor &&
                   PCI_PRODUCT(reg) == mpi->subproduct)
                       return (mpi);
       }

       return (NULL);
}

/*
* Match a supported board.
*/
static int
mly_match(device_t parent, cfdata_t cfdata, void *aux)
{

       return (mly_find_ident(aux) != NULL);
}

/*
* Attach a supported board.
*/
static void
mly_attach(device_t parent, device_t self, void *aux)
{
       struct pci_attach_args *pa;
       struct mly_softc *mly;
       struct mly_ioctl_getcontrollerinfo *mi;
       const struct mly_ident *ident;
       pci_chipset_tag_t pc;
       pci_intr_handle_t ih;
       bus_space_handle_t memh, ioh;
       bus_space_tag_t memt, iot;
       pcireg_t reg;
       const char *intrstr;
       int ior, memr, i, rv, state;
       struct scsipi_adapter *adapt;
       struct scsipi_channel *chan;
       char intrbuf[PCI_INTRSTR_LEN];

       mly = device_private(self);
       mly->mly_dv = self;
       pa = aux;
       pc = pa->pa_pc;
       ident = mly_find_ident(pa);
       state = 0;

       mly->mly_dmat = pa->pa_dmat;
       mly->mly_hwif = ident->hwif;

       printf(": Mylex %s\n", ident->desc);

       /*
        * Map the PCI register window.
        */
       memr = -1;
       ior = -1;

       for (i = 0x10; i <= 0x14; i += 4) {
               reg = pci_conf_read(pa->pa_pc, pa->pa_tag, i);

               if (PCI_MAPREG_TYPE(reg) == PCI_MAPREG_TYPE_IO) {
                       if (ior == -1 && PCI_MAPREG_IO_SIZE(reg) != 0)
                               ior = i;
               } else {
                       if (memr == -1 && PCI_MAPREG_MEM_SIZE(reg) != 0)
                               memr = i;
               }
       }

       if (memr != -1)
               if (pci_mapreg_map(pa, memr, PCI_MAPREG_TYPE_MEM, 0,
                   &memt, &memh, NULL, NULL))
                       memr = -1;
       if (ior != -1)
               if (pci_mapreg_map(pa, ior, PCI_MAPREG_TYPE_IO, 0,
                   &iot, &ioh, NULL, NULL))
                       ior = -1;

       if (memr != -1) {
               mly->mly_iot = memt;
               mly->mly_ioh = memh;
       } else if (ior != -1) {
               mly->mly_iot = iot;
               mly->mly_ioh = ioh;
       } else {
               aprint_error_dev(self, "can't map i/o or memory space\n");
               return;
       }

       /*
        * Enable the device.
        */
       reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
       pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
           reg | PCI_COMMAND_MASTER_ENABLE);

       /*
        * Map and establish the interrupt.
        */
       if (pci_intr_map(pa, &ih)) {
               aprint_error_dev(self, "can't map interrupt\n");
               return;
       }
       intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
       mly->mly_ih = pci_intr_establish_xname(pc, ih, IPL_BIO, mly_intr, mly,
           device_xname(self));
       if (mly->mly_ih == NULL) {
               aprint_error_dev(self, "can't establish interrupt");
               if (intrstr != NULL)
                       aprint_error(" at %s", intrstr);
               aprint_error("\n");
               return;
       }

       if (intrstr != NULL)
               aprint_normal_dev(self, "interrupting at %s\n", intrstr);

       /*
        * Take care of interface-specific tasks.
        */
       switch (mly->mly_hwif) {
       case MLY_HWIF_I960RX:
               mly->mly_doorbell_true = 0x00;
               mly->mly_cmd_mailbox = MLY_I960RX_COMMAND_MAILBOX;
               mly->mly_status_mailbox = MLY_I960RX_STATUS_MAILBOX;
               mly->mly_idbr = MLY_I960RX_IDBR;
               mly->mly_odbr = MLY_I960RX_ODBR;
               mly->mly_error_status = MLY_I960RX_ERROR_STATUS;
               mly->mly_interrupt_status = MLY_I960RX_INTERRUPT_STATUS;
               mly->mly_interrupt_mask = MLY_I960RX_INTERRUPT_MASK;
               break;

       case MLY_HWIF_STRONGARM:
               mly->mly_doorbell_true = 0xff;
               mly->mly_cmd_mailbox = MLY_STRONGARM_COMMAND_MAILBOX;
               mly->mly_status_mailbox = MLY_STRONGARM_STATUS_MAILBOX;
               mly->mly_idbr = MLY_STRONGARM_IDBR;
               mly->mly_odbr = MLY_STRONGARM_ODBR;
               mly->mly_error_status = MLY_STRONGARM_ERROR_STATUS;
               mly->mly_interrupt_status = MLY_STRONGARM_INTERRUPT_STATUS;
               mly->mly_interrupt_mask = MLY_STRONGARM_INTERRUPT_MASK;
               break;
       }

       /*
        * Allocate and map the scatter/gather lists.
        */
       rv = mly_dmamem_alloc(mly, MLY_SGL_SIZE * MLY_MAX_CCBS,
           &mly->mly_sg_dmamap, (void **)&mly->mly_sg,
           &mly->mly_sg_busaddr, &mly->mly_sg_seg);
       if (rv) {
               printf("%s: unable to allocate S/G maps\n",
                   device_xname(self));
               goto bad;
       }
       state++;

       /*
        * Allocate and map the memory mailbox.
        */
       rv = mly_dmamem_alloc(mly, sizeof(struct mly_mmbox),
           &mly->mly_mmbox_dmamap, (void **)&mly->mly_mmbox,
           &mly->mly_mmbox_busaddr, &mly->mly_mmbox_seg);
       if (rv) {
               aprint_error_dev(self, "unable to allocate mailboxes\n");
               goto bad;
       }
       state++;

       /*
        * Initialise per-controller queues.
        */
       SLIST_INIT(&mly->mly_ccb_free);
       SIMPLEQ_INIT(&mly->mly_ccb_queue);

       /*
        * Disable interrupts before we start talking to the controller.
        */
       mly_outb(mly, mly->mly_interrupt_mask, MLY_INTERRUPT_MASK_DISABLE);

       /*
        * Wait for the controller to come ready, handshaking with the
        * firmware if required.  This is typically only necessary on
        * platforms where the controller BIOS does not run.
        */
       if (mly_fwhandshake(mly)) {
               aprint_error_dev(self, "unable to bring controller online\n");
               goto bad;
       }

       /*
        * Allocate initial command buffers, obtain controller feature
        * information, and then reallocate command buffers, since we'll
        * know how many we want.
        */
       if (mly_alloc_ccbs(mly)) {
               aprint_error_dev(self, "unable to allocate CCBs\n");
               goto bad;
       }
       state++;
       if (mly_get_controllerinfo(mly)) {
               aprint_error_dev(self, "unable to retrieve controller info\n");
               goto bad;
       }
       mly_release_ccbs(mly);
       if (mly_alloc_ccbs(mly)) {
               aprint_error_dev(self, "unable to allocate CCBs\n");
               state--;
               goto bad;
       }

       /*
        * Get the current event counter for health purposes, populate the
        * initial health status buffer.
        */
       if (mly_get_eventstatus(mly)) {
               aprint_error_dev(self, "unable to retrieve event status\n");
               goto bad;
       }

       /*
        * Enable memory-mailbox mode.
        */
       if (mly_enable_mmbox(mly)) {
               aprint_error_dev(self, "unable to enable memory mailbox\n");
               goto bad;
       }

       /*
        * Print a little information about the controller.
        */
       mi = mly->mly_controllerinfo;

       printf("%s: %d physical channel%s, firmware %d.%02d-%d-%02d "
           "(%02d%02d%02d%02d), %dMB RAM\n", device_xname(self),
           mi->physical_channels_present,
           (mi->physical_channels_present) > 1 ? "s" : "",
           mi->fw_major, mi->fw_minor, mi->fw_turn, mi->fw_build,
           mi->fw_century, mi->fw_year, mi->fw_month, mi->fw_day,
           le16toh(mi->memory_size));

       /*
        * Register our `shutdownhook'.
        */
       if (mly_sdh == NULL)
               shutdownhook_establish(mly_shutdown, NULL);

       /*
        * Clear any previous BTL information.  For each bus that scsipi
        * wants to scan, we'll receive the SCBUSIOLLSCAN ioctl and retrieve
        * all BTL info at that point.
        */
       memset(&mly->mly_btl, 0, sizeof(mly->mly_btl));

       mly->mly_nchans = mly->mly_controllerinfo->physical_channels_present +
           mly->mly_controllerinfo->virtual_channels_present;

       /*
        * Attach to scsipi.
        */
       adapt = &mly->mly_adapt;
       memset(adapt, 0, sizeof(*adapt));
       adapt->adapt_dev = self;
       adapt->adapt_nchannels = mly->mly_nchans;
       adapt->adapt_openings = mly->mly_ncmds - MLY_CCBS_RESV;
       adapt->adapt_max_periph = mly->mly_ncmds - MLY_CCBS_RESV;
       adapt->adapt_request = mly_scsipi_request;
       adapt->adapt_minphys = mly_scsipi_minphys;
       adapt->adapt_ioctl = mly_scsipi_ioctl;

       for (i = 0; i < mly->mly_nchans; i++) {
               chan = &mly->mly_chans[i];
               memset(chan, 0, sizeof(*chan));
               chan->chan_adapter = adapt;
               chan->chan_bustype = &scsi_bustype;
               chan->chan_channel = i;
               chan->chan_ntargets = MLY_MAX_TARGETS;
               chan->chan_nluns = MLY_MAX_LUNS;
               chan->chan_id = mly->mly_controllerparam->initiator_id;
               chan->chan_flags = SCSIPI_CHAN_NOSETTLE;
               config_found(self, chan, scsiprint, CFARGS_NONE);
       }

       /*
        * Now enable interrupts...
        */
       mly_outb(mly, mly->mly_interrupt_mask, MLY_INTERRUPT_MASK_ENABLE);

       /*
        * Finally, create our monitoring thread.
        */
       mly->mly_state |= MLY_STATE_INITOK;
       rv = kthread_create(PRI_NONE, 0, NULL, mly_thread, mly,
           &mly->mly_thread, "%s", device_xname(self));
       if (rv != 0)
               aprint_error_dev(self, "unable to create thread (%d)\n", rv);
       return;

bad:
       if (state > 2)
               mly_release_ccbs(mly);
       if (state > 1)
               mly_dmamem_free(mly, sizeof(struct mly_mmbox),
                   mly->mly_mmbox_dmamap, (void *)mly->mly_mmbox,
                   &mly->mly_mmbox_seg);
       if (state > 0)
               mly_dmamem_free(mly, MLY_SGL_SIZE * MLY_MAX_CCBS,
                   mly->mly_sg_dmamap, (void *)mly->mly_sg,
                   &mly->mly_sg_seg);
}

/*
* Scan all possible devices on the specified channel.
*/
static void
mly_scan_channel(struct mly_softc *mly, int bus)
{
       int s, target;

       for (target = 0; target < MLY_MAX_TARGETS; target++) {
               s = splbio();
               if (!mly_scan_btl(mly, bus, target)) {
                       tsleep(&mly->mly_btl[bus][target], PRIBIO, "mlyscan",
                           0);
               }
               splx(s);
       }
}

/*
* Shut down all configured `mly' devices.
*/
static void
mly_shutdown(void *cookie)
{
       struct mly_softc *mly;
       int i;

       for (i = 0; i < mly_cd.cd_ndevs; i++) {
               if ((mly = device_lookup_private(&mly_cd, i)) == NULL)
                       continue;

               if (mly_flush(mly))
                       aprint_error_dev(mly->mly_dv, "unable to flush cache\n");
       }
}

/*
* Fill in the mly_controllerinfo and mly_controllerparam fields in the
* softc.
*/
static int
mly_get_controllerinfo(struct mly_softc *mly)
{
       struct mly_cmd_ioctl mci;
       int rv;

       /*
        * Build the getcontrollerinfo ioctl and send it.
        */
       memset(&mci, 0, sizeof(mci));
       mci.sub_ioctl = MDACIOCTL_GETCONTROLLERINFO;
       rv = mly_ioctl(mly, &mci, (void **)&mly->mly_controllerinfo,
           sizeof(*mly->mly_controllerinfo), NULL, NULL);
       if (rv != 0)
               return (rv);

       /*
        * Build the getcontrollerparameter ioctl and send it.
        */
       memset(&mci, 0, sizeof(mci));
       mci.sub_ioctl = MDACIOCTL_GETCONTROLLERPARAMETER;
       rv = mly_ioctl(mly, &mci, (void **)&mly->mly_controllerparam,
           sizeof(*mly->mly_controllerparam), NULL, NULL);

       return (rv);
}

/*
* Rescan a device, possibly as a consequence of getting an event which
* suggests that it may have changed.  Must be called with interrupts
* blocked.
*/
static int
mly_scan_btl(struct mly_softc *mly, int bus, int target)
{
       struct mly_ccb *mc;
       struct mly_cmd_ioctl *mci;
       int rv;

       if (target == mly->mly_controllerparam->initiator_id) {
               mly->mly_btl[bus][target].mb_flags = MLY_BTL_PROTECTED;
               return (EIO);
       }

       /* Don't re-scan if a scan is already in progress. */
       if ((mly->mly_btl[bus][target].mb_flags & MLY_BTL_SCANNING) != 0)
               return (EBUSY);

       /* Get a command. */
       if ((rv = mly_ccb_alloc(mly, &mc)) != 0)
               return (rv);

       /* Set up the data buffer. */
       mc->mc_data = malloc(sizeof(union mly_devinfo),
           M_DEVBUF, M_NOWAIT|M_ZERO);

       mc->mc_flags |= MLY_CCB_DATAIN;
       mc->mc_complete = mly_complete_rescan;

       /*
        * Build the ioctl.
        */
       mci = (struct mly_cmd_ioctl *)&mc->mc_packet->ioctl;
       mci->opcode = MDACMD_IOCTL;
       mci->timeout = 30 | MLY_TIMEOUT_SECONDS;
       memset(&mci->param, 0, sizeof(mci->param));

       if (MLY_BUS_IS_VIRTUAL(mly, bus)) {
               mc->mc_length = sizeof(struct mly_ioctl_getlogdevinfovalid);
               mci->data_size = htole32(mc->mc_length);
               mci->sub_ioctl = MDACIOCTL_GETLOGDEVINFOVALID;
               _lto3l(MLY_LOGADDR(0, MLY_LOGDEV_ID(mly, bus, target)),
                   mci->addr);
       } else {
               mc->mc_length = sizeof(struct mly_ioctl_getphysdevinfovalid);
               mci->data_size = htole32(mc->mc_length);
               mci->sub_ioctl = MDACIOCTL_GETPHYSDEVINFOVALID;
               _lto3l(MLY_PHYADDR(0, bus, target, 0), mci->addr);
       }

       /*
        * Dispatch the command.
        */
       if ((rv = mly_ccb_map(mly, mc)) != 0) {
               free(mc->mc_data, M_DEVBUF);
               mly_ccb_free(mly, mc);
               return(rv);
       }

       mly->mly_btl[bus][target].mb_flags |= MLY_BTL_SCANNING;
       mly_ccb_enqueue(mly, mc);
       return (0);
}

/*
* Handle the completion of a rescan operation.
*/
static void
mly_complete_rescan(struct mly_softc *mly, struct mly_ccb *mc)
{
       struct mly_ioctl_getlogdevinfovalid *ldi;
       struct mly_ioctl_getphysdevinfovalid *pdi;
       struct mly_cmd_ioctl *mci;
       struct mly_btl btl, *btlp;
       struct scsipi_xfer_mode xm;
       int bus, target, rescan;
       u_int tmp;

       mly_ccb_unmap(mly, mc);

       /*
        * Recover the bus and target from the command.  We need these even
        * in the case where we don't have a useful response.
        */
       mci = (struct mly_cmd_ioctl *)&mc->mc_packet->ioctl;
       tmp = _3ltol(mci->addr);
       rescan = 0;

       if (mci->sub_ioctl == MDACIOCTL_GETLOGDEVINFOVALID) {
               bus = MLY_LOGDEV_BUS(mly, MLY_LOGADDR_DEV(tmp));
               target = MLY_LOGDEV_TARGET(mly, MLY_LOGADDR_DEV(tmp));
       } else {
               bus = MLY_PHYADDR_CHANNEL(tmp);
               target = MLY_PHYADDR_TARGET(tmp);
       }

       btlp = &mly->mly_btl[bus][target];

       /* The default result is 'no device'. */
       memset(&btl, 0, sizeof(btl));
       btl.mb_flags = MLY_BTL_PROTECTED;

       /* If the rescan completed OK, we have possibly-new BTL data. */
       if (mc->mc_status != 0)
               goto out;

       if (mc->mc_length == sizeof(*ldi)) {
               ldi = (struct mly_ioctl_getlogdevinfovalid *)mc->mc_data;
               tmp = le32toh(ldi->logical_device_number);

               if (MLY_LOGDEV_BUS(mly, tmp) != bus ||
                   MLY_LOGDEV_TARGET(mly, tmp) != target) {
#ifdef MLYDEBUG
                       printf("%s: WARNING: BTL rescan (logical) for %d:%d "
                           "returned data for %d:%d instead\n",
                          device_xname(mly->mly_dv), bus, target,
                          MLY_LOGDEV_BUS(mly, tmp),
                          MLY_LOGDEV_TARGET(mly, tmp));
#endif
                       goto out;
               }

               btl.mb_flags = MLY_BTL_LOGICAL | MLY_BTL_TQING;
               btl.mb_type = ldi->raid_level;
               btl.mb_state = ldi->state;
       } else if (mc->mc_length == sizeof(*pdi)) {
               pdi = (struct mly_ioctl_getphysdevinfovalid *)mc->mc_data;

               if (pdi->channel != bus || pdi->target != target) {
#ifdef MLYDEBUG
                       printf("%s: WARNING: BTL rescan (physical) for %d:%d "
                           " returned data for %d:%d instead\n",
                          device_xname(mly->mly_dv),
                          bus, target, pdi->channel, pdi->target);
#endif
                       goto out;
               }

               btl.mb_flags = MLY_BTL_PHYSICAL;
               btl.mb_type = MLY_DEVICE_TYPE_PHYSICAL;
               btl.mb_state = pdi->state;
               btl.mb_speed = pdi->speed;
               btl.mb_width = pdi->width;

               if (pdi->state != MLY_DEVICE_STATE_UNCONFIGURED)
                       btl.mb_flags |= MLY_BTL_PROTECTED;
               if (pdi->command_tags != 0)
                       btl.mb_flags |= MLY_BTL_TQING;
       } else {
               printf("%s: BTL rescan result invalid\n", device_xname(mly->mly_dv));
               goto out;
       }

       /* Decide whether we need to rescan the device. */
       if (btl.mb_flags != btlp->mb_flags ||
           btl.mb_speed != btlp->mb_speed ||
           btl.mb_width != btlp->mb_width)
               rescan = 1;

out:
       *btlp = btl;

       if (rescan && (btl.mb_flags & MLY_BTL_PROTECTED) == 0) {
               xm.xm_target = target;
               mly_get_xfer_mode(mly, bus, &xm);
               /* XXX SCSI mid-layer rescan goes here. */
       }

       /* Wake anybody waiting on the device to be rescanned. */
       wakeup(btlp);

       free(mc->mc_data, M_DEVBUF);
       mly_ccb_free(mly, mc);
}

/*
* Get the current health status and set the 'next event' counter to suit.
*/
static int
mly_get_eventstatus(struct mly_softc *mly)
{
       struct mly_cmd_ioctl mci;
       struct mly_health_status *mh;
       int rv;

       /* Build the gethealthstatus ioctl and send it. */
       memset(&mci, 0, sizeof(mci));
       mh = NULL;
       mci.sub_ioctl = MDACIOCTL_GETHEALTHSTATUS;

       rv = mly_ioctl(mly, &mci, (void *)&mh, sizeof(*mh), NULL, NULL);
       if (rv)
               return (rv);

       /* Get the event counter. */
       mly->mly_event_change = le32toh(mh->change_counter);
       mly->mly_event_waiting = le32toh(mh->next_event);
       mly->mly_event_counter = le32toh(mh->next_event);

       /* Save the health status into the memory mailbox */
       memcpy(&mly->mly_mmbox->mmm_health.status, mh, sizeof(*mh));

       bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
           offsetof(struct mly_mmbox, mmm_health),
           sizeof(mly->mly_mmbox->mmm_health),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       free(mh, M_DEVBUF);
       return (0);
}

/*
* Enable memory mailbox mode.
*/
static int
mly_enable_mmbox(struct mly_softc *mly)
{
       struct mly_cmd_ioctl mci;
       u_int8_t *sp;
       u_int64_t tmp;
       int rv;

       /* Build the ioctl and send it. */
       memset(&mci, 0, sizeof(mci));
       mci.sub_ioctl = MDACIOCTL_SETMEMORYMAILBOX;

       /* Set buffer addresses. */
       tmp = mly->mly_mmbox_busaddr + offsetof(struct mly_mmbox, mmm_command);
       mci.param.setmemorymailbox.command_mailbox_physaddr = htole64(tmp);

       tmp = mly->mly_mmbox_busaddr + offsetof(struct mly_mmbox, mmm_status);
       mci.param.setmemorymailbox.status_mailbox_physaddr = htole64(tmp);

       tmp = mly->mly_mmbox_busaddr + offsetof(struct mly_mmbox, mmm_health);
       mci.param.setmemorymailbox.health_buffer_physaddr = htole64(tmp);

       /* Set buffer sizes - abuse of data_size field is revolting. */
       sp = (u_int8_t *)&mci.data_size;
       sp[0] = (sizeof(union mly_cmd_packet) * MLY_MMBOX_COMMANDS) >> 10;
       sp[1] = (sizeof(union mly_status_packet) * MLY_MMBOX_STATUS) >> 10;
       mci.param.setmemorymailbox.health_buffer_size =
           sizeof(union mly_health_region) >> 10;

       rv = mly_ioctl(mly, &mci, NULL, 0, NULL, NULL);
       if (rv)
               return (rv);

       mly->mly_state |= MLY_STATE_MMBOX_ACTIVE;
       return (0);
}

/*
* Flush all pending I/O from the controller.
*/
static int
mly_flush(struct mly_softc *mly)
{
       struct mly_cmd_ioctl mci;

       /* Build the ioctl */
       memset(&mci, 0, sizeof(mci));
       mci.sub_ioctl = MDACIOCTL_FLUSHDEVICEDATA;
       mci.param.deviceoperation.operation_device =
           MLY_OPDEVICE_PHYSICAL_CONTROLLER;

       /* Pass it off to the controller */
       return (mly_ioctl(mly, &mci, NULL, 0, NULL, NULL));
}

/*
* Perform an ioctl command.
*
* If (data) is not NULL, the command requires data transfer to the
* controller.  If (*data) is NULL the command requires data transfer from
* the controller, and we will allocate a buffer for it.
*/
static int
mly_ioctl(struct mly_softc *mly, struct mly_cmd_ioctl *ioctl, void **data,
         size_t datasize, void *sense_buffer,
         size_t *sense_length)
{
       struct mly_ccb *mc;
       struct mly_cmd_ioctl *mci;
       u_int8_t status;
       int rv;

       mc = NULL;
       if ((rv = mly_ccb_alloc(mly, &mc)) != 0)
               goto bad;

       /*
        * Copy the ioctl structure, but save some important fields and then
        * fixup.
        */
       mci = &mc->mc_packet->ioctl;
       ioctl->sense_buffer_address = htole64(mci->sense_buffer_address);
       ioctl->maximum_sense_size = mci->maximum_sense_size;
       *mci = *ioctl;
       mci->opcode = MDACMD_IOCTL;
       mci->timeout = 30 | MLY_TIMEOUT_SECONDS;

       /* Handle the data buffer. */
       if (data != NULL) {
               if (*data == NULL) {
                       /* Allocate data buffer */
                       mc->mc_data = malloc(datasize, M_DEVBUF, M_NOWAIT);
                       mc->mc_flags |= MLY_CCB_DATAIN;
               } else {
                       mc->mc_data = *data;
                       mc->mc_flags |= MLY_CCB_DATAOUT;
               }
               mc->mc_length = datasize;
               mc->mc_packet->generic.data_size = htole32(datasize);
       }

       /* Run the command. */
       if (datasize > 0)
               if ((rv = mly_ccb_map(mly, mc)) != 0)
                       goto bad;
       rv = mly_ccb_poll(mly, mc, 30000);
       if (datasize > 0)
               mly_ccb_unmap(mly, mc);
       if (rv != 0)
               goto bad;

       /* Clean up and return any data. */
       status = mc->mc_status;

       if (status != 0)
               printf("mly_ioctl: command status %d\n", status);

       if (mc->mc_sense > 0 && sense_buffer != NULL) {
               memcpy(sense_buffer, mc->mc_packet, mc->mc_sense);
               *sense_length = mc->mc_sense;
               goto bad;
       }

       /* Should we return a data pointer? */
       if (data != NULL && *data == NULL)
               *data = mc->mc_data;

       /* Command completed OK. */
       rv = (status != 0 ? EIO : 0);

bad:
       if (mc != NULL) {
               /* Do we need to free a data buffer we allocated? */
               if (rv != 0 && mc->mc_data != NULL &&
                   (data == NULL || *data == NULL))
                       free(mc->mc_data, M_DEVBUF);
               mly_ccb_free(mly, mc);
       }

       return (rv);
}

/*
* Check for event(s) outstanding in the controller.
*/
static void
mly_check_event(struct mly_softc *mly)
{

       bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
           offsetof(struct mly_mmbox, mmm_health),
           sizeof(mly->mly_mmbox->mmm_health),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

       /*
        * The controller may have updated the health status information, so
        * check for it here.  Note that the counters are all in host
        * memory, so this check is very cheap.  Also note that we depend on
        * checking on completion
        */
       if (le32toh(mly->mly_mmbox->mmm_health.status.change_counter) !=
           mly->mly_event_change) {
               mly->mly_event_change =
                   le32toh(mly->mly_mmbox->mmm_health.status.change_counter);
               mly->mly_event_waiting =
                   le32toh(mly->mly_mmbox->mmm_health.status.next_event);

               /* Wake up anyone that might be interested in this. */
               wakeup(&mly->mly_event_change);
       }

       bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
           offsetof(struct mly_mmbox, mmm_health),
           sizeof(mly->mly_mmbox->mmm_health),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       if (mly->mly_event_counter != mly->mly_event_waiting)
               mly_fetch_event(mly);
}

/*
* Fetch one event from the controller.  If we fail due to resource
* starvation, we'll be retried the next time a command completes.
*/
static void
mly_fetch_event(struct mly_softc *mly)
{
       struct mly_ccb *mc;
       struct mly_cmd_ioctl *mci;
       int s;
       u_int32_t event;

       /* Get a command. */
       if (mly_ccb_alloc(mly, &mc))
               return;

       /* Set up the data buffer. */
       mc->mc_data = malloc(sizeof(struct mly_event), M_DEVBUF,
           M_NOWAIT|M_ZERO);

       mc->mc_length = sizeof(struct mly_event);
       mc->mc_flags |= MLY_CCB_DATAIN;
       mc->mc_complete = mly_complete_event;

       /*
        * Get an event number to fetch.  It's possible that we've raced
        * with another context for the last event, in which case there will
        * be no more events.
        */
       s = splbio();
       if (mly->mly_event_counter == mly->mly_event_waiting) {
               splx(s);
               free(mc->mc_data, M_DEVBUF);
               mly_ccb_free(mly, mc);
               return;
       }
       event = mly->mly_event_counter++;
       splx(s);

       /*
        * Build the ioctl.
        *
        * At this point we are committed to sending this request, as it
        * will be the only one constructed for this particular event
        * number.
        */
       mci = (struct mly_cmd_ioctl *)&mc->mc_packet->ioctl;
       mci->opcode = MDACMD_IOCTL;
       mci->data_size = htole32(sizeof(struct mly_event));
       _lto3l(MLY_PHYADDR(0, 0, (event >> 16) & 0xff, (event >> 24) & 0xff),
           mci->addr);
       mci->timeout = 30 | MLY_TIMEOUT_SECONDS;
       mci->sub_ioctl = MDACIOCTL_GETEVENT;
       mci->param.getevent.sequence_number_low = htole16(event & 0xffff);

       /*
        * Submit the command.
        */
       if (mly_ccb_map(mly, mc) != 0)
               goto bad;
       mly_ccb_enqueue(mly, mc);
       return;

bad:
       printf("%s: couldn't fetch event %u\n", device_xname(mly->mly_dv), event);
       free(mc->mc_data, M_DEVBUF);
       mly_ccb_free(mly, mc);
}

/*
* Handle the completion of an event poll.
*/
static void
mly_complete_event(struct mly_softc *mly, struct mly_ccb *mc)
{
       struct mly_event *me;

       me = (struct mly_event *)mc->mc_data;
       mly_ccb_unmap(mly, mc);
       mly_ccb_free(mly, mc);

       /* If the event was successfully fetched, process it. */
       if (mc->mc_status == SCSI_OK)
               mly_process_event(mly, me);
       else
               aprint_error_dev(mly->mly_dv, "unable to fetch event; status = 0x%x\n",
                   mc->mc_status);

       free(me, M_DEVBUF);

       /* Check for another event. */
       mly_check_event(mly);
}

/*
* Process a controller event.  Called with interrupts blocked (i.e., at
* interrupt time).
*/
static void
mly_process_event(struct mly_softc *mly, struct mly_event *me)
{
       struct scsi_sense_data *ssd;
       int bus, target, event, class, action;
       const char *fp, *tp;

       ssd = (struct scsi_sense_data *)&me->sense[0];

       /*
        * Errors can be reported using vendor-unique sense data.  In this
        * case, the event code will be 0x1c (Request sense data present),
        * the sense key will be 0x09 (vendor specific), the MSB of the ASC
        * will be set, and the actual event code will be a 16-bit value
        * comprised of the ASCQ (low byte) and low seven bits of the ASC
        * (low seven bits of the high byte).
        */
       if (le32toh(me->code) == 0x1c &&
           SSD_SENSE_KEY(ssd->flags) == SKEY_VENDOR_SPECIFIC &&
           (ssd->asc & 0x80) != 0) {
               event = ((int)(ssd->asc & ~0x80) << 8) +
                   ssd->ascq;
       } else
               event = le32toh(me->code);

       /* Look up event, get codes. */
       fp = mly_describe_code(mly_table_event, event);

       /* Quiet event? */
       class = fp[0];
#ifdef notyet
       if (isupper(class) && bootverbose)
               class = tolower(class);
#endif

       /* Get action code, text string. */
       action = fp[1];
       tp = fp + 3;

       /*
        * Print some information about the event.
        *
        * This code uses a table derived from the corresponding portion of
        * the Linux driver, and thus the parser is very similar.
        */
       switch (class) {
       case 'p':
               /*
                * Error on physical drive.
                */
               printf("%s: physical device %d:%d %s\n", device_xname(mly->mly_dv),
                   me->channel, me->target, tp);
               if (action == 'r')
                       mly->mly_btl[me->channel][me->target].mb_flags |=
                           MLY_BTL_RESCAN;
               break;

       case 'l':
       case 'm':
               /*
                * Error on logical unit, or message about logical unit.
                */
               bus = MLY_LOGDEV_BUS(mly, me->lun);
               target = MLY_LOGDEV_TARGET(mly, me->lun);
               printf("%s: logical device %d:%d %s\n", device_xname(mly->mly_dv),
                   bus, target, tp);
               if (action == 'r')
                       mly->mly_btl[bus][target].mb_flags |= MLY_BTL_RESCAN;
               break;

       case 's':
               /*
                * Report of sense data.
                */
               if ((SSD_SENSE_KEY(ssd->flags) == SKEY_NO_SENSE ||
                    SSD_SENSE_KEY(ssd->flags) == SKEY_NOT_READY) &&
                   ssd->asc == 0x04 &&
                   (ssd->ascq == 0x01 ||
                    ssd->ascq == 0x02)) {
                       /* Ignore NO_SENSE or NOT_READY in one case */
                       break;
               }

               /*
                * XXX Should translate this if SCSIVERBOSE.
                */
               printf("%s: physical device %d:%d %s\n", device_xname(mly->mly_dv),
                   me->channel, me->target, tp);
               printf("%s:  sense key %d  asc %02x  ascq %02x\n",
                   device_xname(mly->mly_dv), SSD_SENSE_KEY(ssd->flags),
                   ssd->asc, ssd->ascq);
               printf("%s:  info %x%x%x%x  csi %x%x%x%x\n",
                   device_xname(mly->mly_dv), ssd->info[0], ssd->info[1],
                   ssd->info[2], ssd->info[3], ssd->csi[0],
                   ssd->csi[1], ssd->csi[2],
                   ssd->csi[3]);
               if (action == 'r')
                       mly->mly_btl[me->channel][me->target].mb_flags |=
                           MLY_BTL_RESCAN;
               break;

       case 'e':
               printf("%s: ", device_xname(mly->mly_dv));
               printf(tp, me->target, me->lun);
               break;

       case 'c':
               printf("%s: controller %s\n", device_xname(mly->mly_dv), tp);
               break;

       case '?':
               printf("%s: %s - %d\n", device_xname(mly->mly_dv), tp, event);
               break;

       default:
               /* Probably a 'noisy' event being ignored. */
               break;
       }
}

/*
* Perform periodic activities.
*/
static void
mly_thread(void *cookie)
{
       struct mly_softc *mly;
       struct mly_btl *btl;
       int s, bus, target, done;

       mly = (struct mly_softc *)cookie;

       for (;;) {
               /* Check for new events. */
               mly_check_event(mly);

               /* Re-scan up to 1 device. */
               s = splbio();
               done = 0;
               for (bus = 0; bus < mly->mly_nchans && !done; bus++) {
                       for (target = 0; target < MLY_MAX_TARGETS; target++) {
                               /* Perform device rescan? */
                               btl = &mly->mly_btl[bus][target];
                               if ((btl->mb_flags & MLY_BTL_RESCAN) != 0) {
                                       btl->mb_flags ^= MLY_BTL_RESCAN;
                                       mly_scan_btl(mly, bus, target);
                                       done = 1;
                                       break;
                               }
                       }
               }
               splx(s);

               /* Sleep for N seconds. */
               tsleep(mly_thread, PWAIT, "mlyzzz",
                   hz * MLY_PERIODIC_INTERVAL);
       }
}

/*
* Submit a command to the controller and poll on completion.  Return
* non-zero on timeout.
*/
static int
mly_ccb_poll(struct mly_softc *mly, struct mly_ccb *mc, int timo)
{
       int rv;

       if ((rv = mly_ccb_submit(mly, mc)) != 0)
               return (rv);

       for (timo *= 10; timo != 0; timo--) {
               if ((mc->mc_flags & MLY_CCB_COMPLETE) != 0)
                       break;
               mly_intr(mly);
               DELAY(100);
       }

       return (timo == 0);
}

/*
* Submit a command to the controller and sleep on completion.  Return
* non-zero on timeout.
*/
static int
mly_ccb_wait(struct mly_softc *mly, struct mly_ccb *mc, int timo)
{
       int rv, s;

       mly_ccb_enqueue(mly, mc);

       s = splbio();
       if ((mc->mc_flags & MLY_CCB_COMPLETE) != 0) {
               splx(s);
               return (0);
       }
       rv = tsleep(mc, PRIBIO, "mlywccb", timo * hz / 1000);
       splx(s);

       return (rv);
}

/*
* If a CCB is specified, enqueue it.  Pull CCBs off the software queue in
* the order that they were enqueued and try to submit their command blocks
* to the controller for execution.
*/
void
mly_ccb_enqueue(struct mly_softc *mly, struct mly_ccb *mc)
{
       int s;

       s = splbio();

       if (mc != NULL)
               SIMPLEQ_INSERT_TAIL(&mly->mly_ccb_queue, mc, mc_link.simpleq);

       while ((mc = SIMPLEQ_FIRST(&mly->mly_ccb_queue)) != NULL) {
               if (mly_ccb_submit(mly, mc))
                       break;
               SIMPLEQ_REMOVE_HEAD(&mly->mly_ccb_queue, mc_link.simpleq);
       }

       splx(s);
}

/*
* Deliver a command to the controller.
*/
static int
mly_ccb_submit(struct mly_softc *mly, struct mly_ccb *mc)
{
       union mly_cmd_packet *pkt;
       int s, off;

       mc->mc_packet->generic.command_id = htole16(mc->mc_slot);

       bus_dmamap_sync(mly->mly_dmat, mly->mly_pkt_dmamap,
           mc->mc_packetphys - mly->mly_pkt_busaddr,
           sizeof(union mly_cmd_packet),
           BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

       s = splbio();

       /*
        * Do we have to use the hardware mailbox?
        */
       if ((mly->mly_state & MLY_STATE_MMBOX_ACTIVE) == 0) {
               /*
                * Check to see if the controller is ready for us.
                */
               if (mly_idbr_true(mly, MLY_HM_CMDSENT)) {
                       splx(s);
                       return (EBUSY);
               }

               /*
                * It's ready, send the command.
                */
               mly_outl(mly, mly->mly_cmd_mailbox,
                   (u_int64_t)mc->mc_packetphys & 0xffffffff);
               mly_outl(mly, mly->mly_cmd_mailbox + 4,
                   (u_int64_t)mc->mc_packetphys >> 32);
               mly_outb(mly, mly->mly_idbr, MLY_HM_CMDSENT);
       } else {
               pkt = &mly->mly_mmbox->mmm_command[mly->mly_mmbox_cmd_idx];
               off = (char *)pkt - (char *)mly->mly_mmbox;

               bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
                   off, sizeof(mly->mly_mmbox->mmm_command[0]),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

               /* Check to see if the next index is free yet. */
               if (pkt->mmbox.flag != 0) {
                       splx(s);
                       return (EBUSY);
               }

               /* Copy in new command */
               memcpy(pkt->mmbox.data, mc->mc_packet->mmbox.data,
                   sizeof(pkt->mmbox.data));

               /* Copy flag last. */
               pkt->mmbox.flag = mc->mc_packet->mmbox.flag;

               bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
                   off, sizeof(mly->mly_mmbox->mmm_command[0]),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

               /* Signal controller and update index. */
               mly_outb(mly, mly->mly_idbr, MLY_AM_CMDSENT);
               mly->mly_mmbox_cmd_idx =
                   (mly->mly_mmbox_cmd_idx + 1) % MLY_MMBOX_COMMANDS;
       }

       splx(s);
       return (0);
}

/*
* Pick up completed commands from the controller and handle accordingly.
*/
int
mly_intr(void *cookie)
{
       struct mly_ccb *mc;
       union mly_status_packet *sp;
       u_int16_t slot;
       int forus, off;
       struct mly_softc *mly;

       mly = cookie;
       forus = 0;

       /*
        * Pick up hardware-mailbox commands.
        */
       if (mly_odbr_true(mly, MLY_HM_STSREADY)) {
               slot = mly_inw(mly, mly->mly_status_mailbox);

               if (slot < MLY_SLOT_MAX) {
                       mc = mly->mly_ccbs + (slot - MLY_SLOT_START);
                       mc->mc_status =
                           mly_inb(mly, mly->mly_status_mailbox + 2);
                       mc->mc_sense =
                           mly_inb(mly, mly->mly_status_mailbox + 3);
                       mc->mc_resid =
                           mly_inl(mly, mly->mly_status_mailbox + 4);

                       mly_ccb_complete(mly, mc);
               } else {
                       /* Slot 0xffff may mean "extremely bogus command". */
                       printf("%s: got HM completion for illegal slot %u\n",
                           device_xname(mly->mly_dv), slot);
               }

               /* Unconditionally acknowledge status. */
               mly_outb(mly, mly->mly_odbr, MLY_HM_STSREADY);
               mly_outb(mly, mly->mly_idbr, MLY_HM_STSACK);
               forus = 1;
       }

       /*
        * Pick up memory-mailbox commands.
        */
       if (mly_odbr_true(mly, MLY_AM_STSREADY)) {
               for (;;) {
                       sp = &mly->mly_mmbox->mmm_status[mly->mly_mmbox_sts_idx];
                       off = (char *)sp - (char *)mly->mly_mmbox;

                       bus_dmamap_sync(mly->mly_dmat, mly->mly_mmbox_dmamap,
                           off, sizeof(mly->mly_mmbox->mmm_command[0]),
                           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

                       /* Check for more status. */
                       if (sp->mmbox.flag == 0)
                               break;

                       /* Get slot number. */
                       slot = le16toh(sp->status.command_id);
                       if (slot < MLY_SLOT_MAX) {
                               mc = mly->mly_ccbs + (slot - MLY_SLOT_START);
                               mc->mc_status = sp->status.status;
                               mc->mc_sense = sp->status.sense_length;
                               mc->mc_resid = le32toh(sp->status.residue);
                               mly_ccb_complete(mly, mc);
                       } else {
                               /*
                                * Slot 0xffff may mean "extremely bogus
                                * command".
                                */
                               printf("%s: got AM completion for illegal "
                                   "slot %u at %d\n", device_xname(mly->mly_dv),
                                   slot, mly->mly_mmbox_sts_idx);
                       }

                       /* Clear and move to next index. */
                       sp->mmbox.flag = 0;
                       mly->mly_mmbox_sts_idx =
                           (mly->mly_mmbox_sts_idx + 1) % MLY_MMBOX_STATUS;
               }

               /* Acknowledge that we have collected status value(s). */
               mly_outb(mly, mly->mly_odbr, MLY_AM_STSREADY);
               forus = 1;
       }

       /*
        * Run the queue.
        */
       if (forus && ! SIMPLEQ_EMPTY(&mly->mly_ccb_queue))
               mly_ccb_enqueue(mly, NULL);

       return (forus);
}

/*
* Process completed commands
*/
static void
mly_ccb_complete(struct mly_softc *mly, struct mly_ccb *mc)
{
       void (*complete)(struct mly_softc *, struct mly_ccb *);

       bus_dmamap_sync(mly->mly_dmat, mly->mly_pkt_dmamap,
           mc->mc_packetphys - mly->mly_pkt_busaddr,
           sizeof(union mly_cmd_packet),
           BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

       complete = mc->mc_complete;
       mc->mc_flags |= MLY_CCB_COMPLETE;

       /*
        * Call completion handler or wake up sleeping consumer.
        */
       if (complete != NULL)
               (*complete)(mly, mc);
       else
               wakeup(mc);
}

/*
* Allocate a command.
*/
int
mly_ccb_alloc(struct mly_softc *mly, struct mly_ccb **mcp)
{
       struct mly_ccb *mc;
       int s;

       s = splbio();
       mc = SLIST_FIRST(&mly->mly_ccb_free);
       if (mc != NULL)
               SLIST_REMOVE_HEAD(&mly->mly_ccb_free, mc_link.slist);
       splx(s);

       *mcp = mc;
       return (mc == NULL ? EAGAIN : 0);
}

/*
* Release a command back to the freelist.
*/
void
mly_ccb_free(struct mly_softc *mly, struct mly_ccb *mc)
{
       int s;

       /*
        * Fill in parts of the command that may cause confusion if a
        * consumer doesn't when we are later allocated.
        */
       mc->mc_data = NULL;
       mc->mc_flags = 0;
       mc->mc_complete = NULL;
       mc->mc_private = NULL;
       mc->mc_packet->generic.command_control = 0;

       /*
        * By default, we set up to overwrite the command packet with sense
        * information.
        */
       mc->mc_packet->generic.sense_buffer_address =
           htole64(mc->mc_packetphys);
       mc->mc_packet->generic.maximum_sense_size =
           sizeof(union mly_cmd_packet);

       s = splbio();
       SLIST_INSERT_HEAD(&mly->mly_ccb_free, mc, mc_link.slist);
       splx(s);
}

/*
* Allocate and initialize command and packet structures.
*
* If the controller supports fewer than MLY_MAX_CCBS commands, limit our
* allocation to that number.  If we don't yet know how many commands the
* controller supports, allocate a very small set (suitable for initialization
* purposes only).
*/
static int
mly_alloc_ccbs(struct mly_softc *mly)
{
       struct mly_ccb *mc;
       int i, rv;

       if (mly->mly_controllerinfo == NULL)
               mly->mly_ncmds = MLY_CCBS_RESV;
       else {
               i = le16toh(mly->mly_controllerinfo->maximum_parallel_commands);
               mly->mly_ncmds = uimin(MLY_MAX_CCBS, i);
       }

       /*
        * Allocate enough space for all the command packets in one chunk
        * and map them permanently into controller-visible space.
        */
       rv = mly_dmamem_alloc(mly,
           mly->mly_ncmds * sizeof(union mly_cmd_packet),
           &mly->mly_pkt_dmamap, (void **)&mly->mly_pkt,
           &mly->mly_pkt_busaddr, &mly->mly_pkt_seg);
       if (rv)
               return (rv);

       mly->mly_ccbs = malloc(sizeof(struct mly_ccb) * mly->mly_ncmds,
           M_DEVBUF, M_WAITOK|M_ZERO);

       for (i = 0; i < mly->mly_ncmds; i++) {
               mc = mly->mly_ccbs + i;
               mc->mc_slot = MLY_SLOT_START + i;
               mc->mc_packet = mly->mly_pkt + i;
               mc->mc_packetphys = mly->mly_pkt_busaddr +
                   (i * sizeof(union mly_cmd_packet));

               rv = bus_dmamap_create(mly->mly_dmat, MLY_MAX_XFER,
                   MLY_MAX_SEGS, MLY_MAX_XFER, 0,
                   BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
                   &mc->mc_datamap);
               if (rv) {
                       mly_release_ccbs(mly);
                       return (rv);
               }

               mly_ccb_free(mly, mc);
       }

       return (0);
}

/*
* Free all the storage held by commands.
*
* Must be called with all commands on the free list.
*/
static void
mly_release_ccbs(struct mly_softc *mly)
{
       struct mly_ccb *mc;

       /* Throw away command buffer DMA maps. */
       while (mly_ccb_alloc(mly, &mc) == 0)
               bus_dmamap_destroy(mly->mly_dmat, mc->mc_datamap);

       /* Release CCB storage. */
       free(mly->mly_ccbs, M_DEVBUF);

       /* Release the packet storage. */
       mly_dmamem_free(mly, mly->mly_ncmds * sizeof(union mly_cmd_packet),
           mly->mly_pkt_dmamap, (void *)mly->mly_pkt, &mly->mly_pkt_seg);
}

/*
* Map a command into controller-visible space.
*/
static int
mly_ccb_map(struct mly_softc *mly, struct mly_ccb *mc)
{
       struct mly_cmd_generic *gen;
       struct mly_sg_entry *sg;
       bus_dma_segment_t *ds;
       int flg, nseg, rv;

#ifdef DIAGNOSTIC
       /* Don't map more than once. */
       if ((mc->mc_flags & MLY_CCB_MAPPED) != 0)
               panic("mly_ccb_map: already mapped");
       mc->mc_flags |= MLY_CCB_MAPPED;

       /* Does the command have a data buffer? */
       if (mc->mc_data == NULL)
               panic("mly_ccb_map: no data buffer");
#endif

       rv = bus_dmamap_load(mly->mly_dmat, mc->mc_datamap, mc->mc_data,
           mc->mc_length, NULL, BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
           ((mc->mc_flags & MLY_CCB_DATAIN) != 0 ?
           BUS_DMA_READ : BUS_DMA_WRITE));
       if (rv != 0)
               return (rv);

       gen = &mc->mc_packet->generic;

       /*
        * Can we use the transfer structure directly?
        */
       if ((nseg = mc->mc_datamap->dm_nsegs) <= 2) {
               mc->mc_sgoff = -1;
               sg = &gen->transfer.direct.sg[0];
       } else {
               mc->mc_sgoff = (mc->mc_slot - MLY_SLOT_START) *
                   MLY_MAX_SEGS;
               sg = mly->mly_sg + mc->mc_sgoff;
               gen->command_control |= MLY_CMDCTL_EXTENDED_SG_TABLE;
               gen->transfer.indirect.entries[0] = htole16(nseg);
               gen->transfer.indirect.table_physaddr[0] =
                   htole64(mly->mly_sg_busaddr +
                   (mc->mc_sgoff * sizeof(struct mly_sg_entry)));
       }

       /*
        * Fill the S/G table.
        */
       for (ds = mc->mc_datamap->dm_segs; nseg != 0; nseg--, sg++, ds++) {
               sg->physaddr = htole64(ds->ds_addr);
               sg->length = htole64(ds->ds_len);
       }

       /*
        * Sync up the data map.
        */
       if ((mc->mc_flags & MLY_CCB_DATAIN) != 0)
               flg = BUS_DMASYNC_PREREAD;
       else /* if ((mc->mc_flags & MLY_CCB_DATAOUT) != 0) */ {
               gen->command_control |= MLY_CMDCTL_DATA_DIRECTION;
               flg = BUS_DMASYNC_PREWRITE;
       }

       bus_dmamap_sync(mly->mly_dmat, mc->mc_datamap, 0, mc->mc_length, flg);

       /*
        * Sync up the chained S/G table, if we're using one.
        */
       if (mc->mc_sgoff == -1)
               return (0);

       bus_dmamap_sync(mly->mly_dmat, mly->mly_sg_dmamap, mc->mc_sgoff,
           MLY_SGL_SIZE, BUS_DMASYNC_PREWRITE);

       return (0);
}

/*
* Unmap a command from controller-visible space.
*/
static void
mly_ccb_unmap(struct mly_softc *mly, struct mly_ccb *mc)
{
       int flg;

#ifdef DIAGNOSTIC
       if ((mc->mc_flags & MLY_CCB_MAPPED) == 0)
               panic("mly_ccb_unmap: not mapped");
       mc->mc_flags &= ~MLY_CCB_MAPPED;
#endif

       if ((mc->mc_flags & MLY_CCB_DATAIN) != 0)
               flg = BUS_DMASYNC_POSTREAD;
       else /* if ((mc->mc_flags & MLY_CCB_DATAOUT) != 0) */
               flg = BUS_DMASYNC_POSTWRITE;

       bus_dmamap_sync(mly->mly_dmat, mc->mc_datamap, 0, mc->mc_length, flg);
       bus_dmamap_unload(mly->mly_dmat, mc->mc_datamap);

       if (mc->mc_sgoff == -1)
               return;

       bus_dmamap_sync(mly->mly_dmat, mly->mly_sg_dmamap, mc->mc_sgoff,
           MLY_SGL_SIZE, BUS_DMASYNC_POSTWRITE);
}

/*
* Adjust the size of each I/O before it passes to the SCSI layer.
*/
static void
mly_scsipi_minphys(struct buf *bp)
{

       if (bp->b_bcount > MLY_MAX_XFER)
               bp->b_bcount = MLY_MAX_XFER;
       minphys(bp);
}

/*
* Start a SCSI command.
*/
static void
mly_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
                  void *arg)
{
       struct mly_ccb *mc;
       struct mly_cmd_scsi_small *ss;
       struct scsipi_xfer *xs;
       struct scsipi_periph *periph;
       struct mly_softc *mly;
       struct mly_btl *btl;
       int s, tmp;

       mly = device_private(chan->chan_adapter->adapt_dev);

       switch (req) {
       case ADAPTER_REQ_RUN_XFER:
               xs = arg;
               periph = xs->xs_periph;
               btl = &mly->mly_btl[chan->chan_channel][periph->periph_target];
               s = splbio();
               tmp = btl->mb_flags;
               splx(s);

               /*
                * Check for I/O attempt to a protected or non-existent
                * device.
                */
               if ((tmp & MLY_BTL_PROTECTED) != 0) {
                       xs->error = XS_SELTIMEOUT;
                       scsipi_done(xs);
                       break;
               }

#ifdef DIAGNOSTIC
               /* XXX Increase if/when we support large SCSI commands. */
               if (xs->cmdlen > MLY_CMD_SCSI_SMALL_CDB) {
                       printf("%s: cmd too large\n", device_xname(mly->mly_dv));
                       xs->error = XS_DRIVER_STUFFUP;
                       scsipi_done(xs);
                       break;
               }
#endif

               if (mly_ccb_alloc(mly, &mc)) {
                       xs->error = XS_RESOURCE_SHORTAGE;
                       scsipi_done(xs);
                       break;
               }

               /* Build the command. */
               mc->mc_data = xs->data;
               mc->mc_length = xs->datalen;
               mc->mc_complete = mly_scsipi_complete;
               mc->mc_private = xs;

               /* Build the packet for the controller. */
               ss = &mc->mc_packet->scsi_small;
               ss->opcode = MDACMD_SCSI;
#ifdef notdef
               /*
                * XXX FreeBSD does this, but it doesn't fix anything,
                * XXX and appears potentially harmful.
                */
               ss->command_control |= MLY_CMDCTL_DISABLE_DISCONNECT;
#endif

               ss->data_size = htole32(xs->datalen);
               _lto3l(MLY_PHYADDR(0, chan->chan_channel,
                   periph->periph_target, periph->periph_lun), ss->addr);

               if (xs->timeout < 60 * 1000)
                       ss->timeout = xs->timeout / 1000 |
                           MLY_TIMEOUT_SECONDS;
               else if (xs->timeout < 60 * 60 * 1000)
                       ss->timeout = xs->timeout / (60 * 1000) |
                           MLY_TIMEOUT_MINUTES;
               else
                       ss->timeout = xs->timeout / (60 * 60 * 1000) |
                           MLY_TIMEOUT_HOURS;

               ss->maximum_sense_size = sizeof(xs->sense);
               ss->cdb_length = xs->cmdlen;
               memcpy(ss->cdb, xs->cmd, xs->cmdlen);

               if (mc->mc_length != 0) {
                       if ((xs->xs_control & XS_CTL_DATA_OUT) != 0)
                               mc->mc_flags |= MLY_CCB_DATAOUT;
                       else /* if ((xs->xs_control & XS_CTL_DATA_IN) != 0) */
                               mc->mc_flags |= MLY_CCB_DATAIN;

                       if (mly_ccb_map(mly, mc) != 0) {
                               xs->error = XS_DRIVER_STUFFUP;
                               mly_ccb_free(mly, mc);
                               scsipi_done(xs);
                               break;
                       }
               }

               /*
                * Give the command to the controller.
                */
               if ((xs->xs_control & XS_CTL_POLL) != 0) {
                       if (mly_ccb_poll(mly, mc, xs->timeout + 5000)) {
                               xs->error = XS_REQUEUE;
                               if (mc->mc_length != 0)
                                       mly_ccb_unmap(mly, mc);
                               mly_ccb_free(mly, mc);
                               scsipi_done(xs);
                       }
               } else
                       mly_ccb_enqueue(mly, mc);

               break;

       case ADAPTER_REQ_GROW_RESOURCES:
               /*
                * Not supported.
                */
               break;

       case ADAPTER_REQ_SET_XFER_MODE:
               /*
                * We can't change the transfer mode, but at least let
                * scsipi know what the adapter has negotiated.
                */
               mly_get_xfer_mode(mly, chan->chan_channel, arg);
               break;
       }
}

/*
* Handle completion of a SCSI command.
*/
static void
mly_scsipi_complete(struct mly_softc *mly, struct mly_ccb *mc)
{
       struct scsipi_xfer *xs;
       struct scsipi_channel *chan;
       struct scsipi_inquiry_data *inq;
       struct mly_btl *btl;
       int target, sl, s;
       const char *p;

       xs = mc->mc_private;
       xs->status = mc->mc_status;

       /*
        * XXX The `resid' value as returned by the controller appears to be
        * bogus, so we always set it to zero.  Is it perhaps the transfer
        * count?
        */
       xs->resid = 0; /* mc->mc_resid; */

       if (mc->mc_length != 0)
               mly_ccb_unmap(mly, mc);

       switch (mc->mc_status) {
       case SCSI_OK:
               /*
                * In order to report logical device type and status, we
                * overwrite the result of the INQUIRY command to logical
                * devices.
                */
               if (xs->cmd->opcode == INQUIRY) {
                       chan = xs->xs_periph->periph_channel;
                       target = xs->xs_periph->periph_target;
                       btl = &mly->mly_btl[chan->chan_channel][target];

                       s = splbio();
                       if ((btl->mb_flags & MLY_BTL_LOGICAL) != 0) {
                               inq = (struct scsipi_inquiry_data *)xs->data;
                               mly_padstr(inq->vendor, "MYLEX", 8);
                               p = mly_describe_code(mly_table_device_type,
                                   btl->mb_type);
                               mly_padstr(inq->product, p, 16);
                               p = mly_describe_code(mly_table_device_state,
                                   btl->mb_state);
                               mly_padstr(inq->revision, p, 4);
                       }
                       splx(s);
               }

               xs->error = XS_NOERROR;
               break;

       case SCSI_CHECK:
               sl = mc->mc_sense;
               if (sl > sizeof(xs->sense.scsi_sense))
                       sl = sizeof(xs->sense.scsi_sense);
               memcpy(&xs->sense.scsi_sense, mc->mc_packet, sl);
               xs->error = XS_SENSE;
               break;

       case SCSI_BUSY:
       case SCSI_QUEUE_FULL:
               xs->error = XS_BUSY;
               break;

       default:
               printf("%s: unknown SCSI status 0x%x\n",
                   device_xname(mly->mly_dv), xs->status);
               xs->error = XS_DRIVER_STUFFUP;
               break;
       }

       mly_ccb_free(mly, mc);
       scsipi_done(xs);
}

/*
* Notify scsipi about a target's transfer mode.
*/
static void
mly_get_xfer_mode(struct mly_softc *mly, int bus, struct scsipi_xfer_mode *xm)
{
       struct mly_btl *btl;
       int s;

       btl = &mly->mly_btl[bus][xm->xm_target];
       xm->xm_mode = 0;

       s = splbio();

       if ((btl->mb_flags & MLY_BTL_PHYSICAL) != 0) {
               if (btl->mb_speed == 0) {
                       xm->xm_period = 0;
                       xm->xm_offset = 0;
               } else {
                       xm->xm_period = 12;                     /* XXX */
                       xm->xm_offset = 8;                      /* XXX */
                       xm->xm_mode |= PERIPH_CAP_SYNC;         /* XXX */
               }

               switch (btl->mb_width) {
               case 32:
                       xm->xm_mode = PERIPH_CAP_WIDE32;
                       break;
               case 16:
                       xm->xm_mode = PERIPH_CAP_WIDE16;
                       break;
               default:
                       xm->xm_mode = 0;
                       break;
               }
       } else /* ((btl->mb_flags & MLY_BTL_LOGICAL) != 0) */ {
               xm->xm_mode = PERIPH_CAP_WIDE16 | PERIPH_CAP_SYNC;
               xm->xm_period = 12;
               xm->xm_offset = 8;
       }

       if ((btl->mb_flags & MLY_BTL_TQING) != 0)
               xm->xm_mode |= PERIPH_CAP_TQING;

       splx(s);

       scsipi_async_event(&mly->mly_chans[bus], ASYNC_EVENT_XFER_MODE, xm);
}

/*
* ioctl hook; used here only to initiate low-level rescans.
*/
static int
mly_scsipi_ioctl(struct scsipi_channel *chan, u_long cmd, void *data,
   int flag, struct proc *p)
{
       struct mly_softc *mly;
       int rv;

       mly = device_private(chan->chan_adapter->adapt_dev);

       switch (cmd) {
       case SCBUSIOLLSCAN:
               mly_scan_channel(mly, chan->chan_channel);
               rv = 0;
               break;
       default:
               rv = ENOTTY;
               break;
       }

       return (rv);
}

/*
* Handshake with the firmware while the card is being initialized.
*/
static int
mly_fwhandshake(struct mly_softc *mly)
{
       u_int8_t error;
       int spinup;

       spinup = 0;

       /* Set HM_STSACK and let the firmware initialize. */
       mly_outb(mly, mly->mly_idbr, MLY_HM_STSACK);
       DELAY(1000);    /* too short? */

       /* If HM_STSACK is still true, the controller is initializing. */
       if (!mly_idbr_true(mly, MLY_HM_STSACK))
               return (0);

       printf("%s: controller initialization started\n",
           device_xname(mly->mly_dv));

       /*
        * Spin waiting for initialization to finish, or for a message to be
        * delivered.
        */
       while (mly_idbr_true(mly, MLY_HM_STSACK)) {
               /* Check for a message */
               if (!mly_error_valid(mly))
                       continue;

               error = mly_inb(mly, mly->mly_error_status) & ~MLY_MSG_EMPTY;
               (void)mly_inb(mly, mly->mly_cmd_mailbox);
               (void)mly_inb(mly, mly->mly_cmd_mailbox + 1);

               switch (error) {
               case MLY_MSG_SPINUP:
                       if (!spinup) {
                               printf("%s: drive spinup in progress\n",
                                   device_xname(mly->mly_dv));
                               spinup = 1;
                       }
                       break;

               case MLY_MSG_RACE_RECOVERY_FAIL:
                       printf("%s: mirror race recovery failed - \n",
                           device_xname(mly->mly_dv));
                       printf("%s: one or more drives offline\n",
                           device_xname(mly->mly_dv));
                       break;

               case MLY_MSG_RACE_IN_PROGRESS:
                       printf("%s: mirror race recovery in progress\n",
                           device_xname(mly->mly_dv));
                       break;

               case MLY_MSG_RACE_ON_CRITICAL:
                       printf("%s: mirror race recovery on critical drive\n",
                           device_xname(mly->mly_dv));
                       break;

               case MLY_MSG_PARITY_ERROR:
                       printf("%s: FATAL MEMORY PARITY ERROR\n",
                           device_xname(mly->mly_dv));
                       return (ENXIO);

               default:
                       printf("%s: unknown initialization code 0x%x\n",
                           device_xname(mly->mly_dv), error);
                       break;
               }
       }

       return (0);
}

/*
* Space-fill a character string
*/
static void
mly_padstr(char *dst, const char *src, int len)
{

       while (len-- > 0) {
               if (*src != '\0')
                       *dst++ = *src++;
               else
                       *dst++ = ' ';
       }
}

/*
* Allocate DMA safe memory.
*/
static int
mly_dmamem_alloc(struct mly_softc *mly, int size, bus_dmamap_t *dmamap,
                void **kva, bus_addr_t *paddr, bus_dma_segment_t *seg)
{
       int rseg, rv, state;

       state = 0;

       if ((rv = bus_dmamem_alloc(mly->mly_dmat, size, PAGE_SIZE, 0,
           seg, 1, &rseg, BUS_DMA_NOWAIT)) != 0) {
               aprint_error_dev(mly->mly_dv, "dmamem_alloc = %d\n", rv);
               goto bad;
       }

       state++;

       if ((rv = bus_dmamem_map(mly->mly_dmat, seg, 1, size, kva,
           BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) != 0) {
               aprint_error_dev(mly->mly_dv, "dmamem_map = %d\n", rv);
               goto bad;
       }

       state++;

       if ((rv = bus_dmamap_create(mly->mly_dmat, size, size, 1, 0,
           BUS_DMA_NOWAIT, dmamap)) != 0) {
               aprint_error_dev(mly->mly_dv, "dmamap_create = %d\n", rv);
               goto bad;
       }

       state++;

       if ((rv = bus_dmamap_load(mly->mly_dmat, *dmamap, *kva, size,
           NULL, BUS_DMA_NOWAIT)) != 0) {
               aprint_error_dev(mly->mly_dv, "dmamap_load = %d\n", rv);
               goto bad;
       }

       *paddr = (*dmamap)->dm_segs[0].ds_addr;
       memset(*kva, 0, size);
       return (0);

bad:
       if (state > 2)
               bus_dmamap_destroy(mly->mly_dmat, *dmamap);
       if (state > 1)
               bus_dmamem_unmap(mly->mly_dmat, *kva, size);
       if (state > 0)
               bus_dmamem_free(mly->mly_dmat, seg, 1);

       return (rv);
}

/*
* Free DMA safe memory.
*/
static void
mly_dmamem_free(struct mly_softc *mly, int size, bus_dmamap_t dmamap,
               void *kva, bus_dma_segment_t *seg)
{

       bus_dmamap_unload(mly->mly_dmat, dmamap);
       bus_dmamap_destroy(mly->mly_dmat, dmamap);
       bus_dmamem_unmap(mly->mly_dmat, kva, size);
       bus_dmamem_free(mly->mly_dmat, seg, 1);
}


/*
* Accept an open operation on the control device.
*/
int
mlyopen(dev_t dev, int flag, int mode, struct lwp *l)
{
       struct mly_softc *mly;

       if ((mly = device_lookup_private(&mly_cd, minor(dev))) == NULL)
               return (ENXIO);
       if ((mly->mly_state & MLY_STATE_INITOK) == 0)
               return (ENXIO);
       if ((mly->mly_state & MLY_STATE_OPEN) != 0)
               return (EBUSY);

       mly->mly_state |= MLY_STATE_OPEN;
       return (0);
}

/*
* Accept the last close on the control device.
*/
int
mlyclose(dev_t dev, int flag, int mode,
   struct lwp *l)
{
       struct mly_softc *mly;

       mly = device_lookup_private(&mly_cd, minor(dev));
       mly->mly_state &= ~MLY_STATE_OPEN;
       return (0);
}

/*
* Handle control operations.
*/
int
mlyioctl(dev_t dev, u_long cmd, void *data, int flag,
   struct lwp *l)
{
       struct mly_softc *mly;
       int rv;

       mly = device_lookup_private(&mly_cd, minor(dev));

       switch (cmd) {
       case MLYIO_COMMAND:
               rv = kauth_authorize_device_passthru(l->l_cred, dev,
                   KAUTH_REQ_DEVICE_RAWIO_PASSTHRU_ALL, data);
               if (rv)
                       break;

               rv = mly_user_command(mly, (void *)data);
               break;
       case MLYIO_HEALTH:
               rv = mly_user_health(mly, (void *)data);
               break;
       default:
               rv = ENOTTY;
               break;
       }

       return (rv);
}

/*
* Execute a command passed in from userspace.
*
* The control structure contains the actual command for the controller, as
* well as the user-space data pointer and data size, and an optional sense
* buffer size/pointer.  On completion, the data size is adjusted to the
* command residual, and the sense buffer size to the size of the returned
* sense data.
*/
static int
mly_user_command(struct mly_softc *mly, struct mly_user_command *uc)
{
       struct mly_ccb  *mc;
       int rv, mapped;

       if ((rv = mly_ccb_alloc(mly, &mc)) != 0)
               return (rv);

       mapped = 0;
       mc->mc_data = NULL;

       /*
        * Handle data size/direction.
        */
       if ((mc->mc_length = abs(uc->DataTransferLength)) != 0) {
               if (mc->mc_length > MAXPHYS) {
                       rv = EINVAL;
                       goto out;
               }

               mc->mc_data = malloc(mc->mc_length, M_DEVBUF, M_WAITOK);
               if (mc->mc_data == NULL) {
                       rv = ENOMEM;
                       goto out;
               }

               if (uc->DataTransferLength > 0) {
                       mc->mc_flags |= MLY_CCB_DATAIN;
                       memset(mc->mc_data, 0, mc->mc_length);
               }

               if (uc->DataTransferLength < 0) {
                       mc->mc_flags |= MLY_CCB_DATAOUT;
                       rv = copyin(uc->DataTransferBuffer, mc->mc_data,
                           mc->mc_length);
                       if (rv != 0)
                               goto out;
               }

               if ((rv = mly_ccb_map(mly, mc)) != 0)
                       goto out;
               mapped = 1;
       }

       /* Copy in the command and execute it. */
       memcpy(mc->mc_packet, &uc->CommandMailbox, sizeof(uc->CommandMailbox));

       if ((rv = mly_ccb_wait(mly, mc, 60000)) != 0)
               goto out;

       /* Return the data to userspace. */
       if (uc->DataTransferLength > 0) {
               rv = copyout(mc->mc_data, uc->DataTransferBuffer,
                   mc->mc_length);
               if (rv != 0)
                       goto out;
       }

       /* Return the sense buffer to userspace. */
       if (uc->RequestSenseLength > 0 && mc->mc_sense > 0) {
               rv = copyout(mc->mc_packet, uc->RequestSenseBuffer,
                   uimin(uc->RequestSenseLength, mc->mc_sense));
               if (rv != 0)
                       goto out;
       }

       /* Return command results to userspace (caller will copy out). */
       uc->DataTransferLength = mc->mc_resid;
       uc->RequestSenseLength = uimin(uc->RequestSenseLength, mc->mc_sense);
       uc->CommandStatus = mc->mc_status;
       rv = 0;

out:
       if (mapped)
               mly_ccb_unmap(mly, mc);
       if (mc->mc_data != NULL)
               free(mc->mc_data, M_DEVBUF);
       mly_ccb_free(mly, mc);

       return (rv);
}

/*
* Return health status to userspace.  If the health change index in the
* user structure does not match that currently exported by the controller,
* we return the current status immediately.  Otherwise, we block until
* either interrupted or new status is delivered.
*/
static int
mly_user_health(struct mly_softc *mly, struct mly_user_health *uh)
{
       struct mly_health_status mh;
       int rv, s;

       /* Fetch the current health status from userspace. */
       rv = copyin(uh->HealthStatusBuffer, &mh, sizeof(mh));
       if (rv != 0)
               return (rv);

       /* spin waiting for a status update */
       s = splbio();
       if (mly->mly_event_change == mh.change_counter)
               rv = tsleep(&mly->mly_event_change, PRIBIO | PCATCH,
                   "mlyhealth", 0);
       splx(s);

       if (rv == 0) {
               /*
                * Copy the controller's health status buffer out (there is
                * a race here if it changes again).
                */
               rv = copyout(&mly->mly_mmbox->mmm_health.status,
                   uh->HealthStatusBuffer, sizeof(uh->HealthStatusBuffer));
       }

       return (rv);
}