/*      $NetBSD: uhci.c,v 1.320 2025/04/26 07:06:53 skrll Exp $ */

/*
* Copyright (c) 1998, 2004, 2011, 2012, 2016, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Lennart Augustsson ([email protected]) at
* Carlstedt Research & Technology, Jared D. McNeill ([email protected]),
* Matthew R. Green ([email protected]) and Nick Hudson.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* USB Universal Host Controller driver.
* Handles e.g. PIIX3 and PIIX4.
*
* UHCI spec: http://www.intel.com/technology/usb/spec.htm
* USB spec: http://www.usb.org/developers/docs/
* PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
*             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uhci.c,v 1.320 2025/04/26 07:06:53 skrll Exp $");

#ifdef _KERNEL_OPT
#include "opt_usb.h"
#endif

#include <sys/param.h>

#include <sys/bus.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/select.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

#include <machine/endian.h>

#include <dev/usb/usb.h>
#include <dev/usb/usbdi.h>
#include <dev/usb/usbdivar.h>
#include <dev/usb/usb_mem.h>

#include <dev/usb/uhcireg.h>
#include <dev/usb/uhcivar.h>
#include <dev/usb/usbroothub.h>
#include <dev/usb/usbhist.h>

/* Use bandwidth reclamation for control transfers. Some devices choke on it. */
/*#define UHCI_CTL_LOOP */

#ifdef UHCI_DEBUG
uhci_softc_t *thesc;
int uhcinoloop = 0;
#endif

#ifdef USB_DEBUG
#ifndef UHCI_DEBUG
#define uhcidebug 0
#else
static int uhcidebug = 0;

SYSCTL_SETUP(sysctl_hw_uhci_setup, "sysctl hw.uhci setup")
{
       int err;
       const struct sysctlnode *rnode;
       const struct sysctlnode *cnode;

       err = sysctl_createv(clog, 0, NULL, &rnode,
           CTLFLAG_PERMANENT, CTLTYPE_NODE, "uhci",
           SYSCTL_DESCR("uhci global controls"),
           NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);

       if (err)
               goto fail;

       /* control debugging printfs */
       err = sysctl_createv(clog, 0, &rnode, &cnode,
           CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
           "debug", SYSCTL_DESCR("Enable debugging output"),
           NULL, 0, &uhcidebug, sizeof(uhcidebug), CTL_CREATE, CTL_EOL);
       if (err)
               goto fail;

       return;
fail:
       aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
}

#endif /* UHCI_DEBUG */
#endif /* USB_DEBUG */

#define DPRINTF(FMT,A,B,C,D)    USBHIST_LOGN(uhcidebug,1,FMT,A,B,C,D)
#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(uhcidebug,N,FMT,A,B,C,D)
#define UHCIHIST_FUNC()         USBHIST_FUNC()
#define UHCIHIST_CALLED(name)   USBHIST_CALLED(uhcidebug)

/*
* The UHCI controller is little endian, so on big endian machines
* the data stored in memory needs to be swapped.
*/

struct uhci_pipe {
       struct usbd_pipe pipe;
       int nexttoggle;

       u_char aborting;
       struct usbd_xfer *abortstart, abortend;

       /* Info needed for different pipe kinds. */
       union {
               /* Control pipe */
               struct {
                       uhci_soft_qh_t *sqh;
                       usb_dma_t reqdma;
                       uhci_soft_td_t *setup;
                       uhci_soft_td_t *stat;
               } ctrl;
               /* Interrupt pipe */
               struct {
                       int npoll;
                       uhci_soft_qh_t **qhs;
               } intr;
               /* Bulk pipe */
               struct {
                       uhci_soft_qh_t *sqh;
               } bulk;
               /* Isochronous pipe */
               struct isoc {
                       uhci_soft_td_t **stds;
                       int next, inuse;
               } isoc;
       };
};

typedef TAILQ_HEAD(ux_completeq, uhci_xfer) ux_completeq_t;

Static void             uhci_globalreset(uhci_softc_t *);
Static usbd_status      uhci_portreset(uhci_softc_t*, int);
Static void             uhci_reset(uhci_softc_t *);
Static usbd_status      uhci_run(uhci_softc_t *, int);
Static uhci_soft_td_t  *uhci_alloc_std(uhci_softc_t *);
Static void             uhci_free_std(uhci_softc_t *, uhci_soft_td_t *);
Static void             uhci_free_std_locked(uhci_softc_t *, uhci_soft_td_t *);
Static uhci_soft_qh_t  *uhci_alloc_sqh(uhci_softc_t *);
Static void             uhci_free_sqh(uhci_softc_t *, uhci_soft_qh_t *);
#if 0
Static void             uhci_enter_ctl_q(uhci_softc_t *, uhci_soft_qh_t *,
                           uhci_intr_info_t *);
Static void             uhci_exit_ctl_q(uhci_softc_t *, uhci_soft_qh_t *);
#endif

#if 0
Static void             uhci_free_std_chain(uhci_softc_t *, uhci_soft_td_t *,
                           uhci_soft_td_t *);
#endif
Static int              uhci_alloc_std_chain(uhci_softc_t *, struct usbd_xfer *,
                           int, int, uhci_soft_td_t **);
Static void             uhci_free_stds(uhci_softc_t *, struct uhci_xfer *);

Static void             uhci_reset_std_chain(uhci_softc_t *, struct usbd_xfer *,
                           int, int, int *, uhci_soft_td_t **);

Static void             uhci_poll_hub(void *);
Static void             uhci_check_intr(uhci_softc_t *, struct uhci_xfer *,
                           ux_completeq_t *);
Static void             uhci_idone(struct uhci_xfer *, ux_completeq_t *);

Static void             uhci_abortx(struct usbd_xfer *);

Static void             uhci_add_ls_ctrl(uhci_softc_t *, uhci_soft_qh_t *);
Static void             uhci_add_hs_ctrl(uhci_softc_t *, uhci_soft_qh_t *);
Static void             uhci_add_bulk(uhci_softc_t *, uhci_soft_qh_t *);
Static void             uhci_remove_ls_ctrl(uhci_softc_t *,uhci_soft_qh_t *);
Static void             uhci_remove_hs_ctrl(uhci_softc_t *,uhci_soft_qh_t *);
Static void             uhci_remove_bulk(uhci_softc_t *,uhci_soft_qh_t *);
Static void             uhci_add_loop(uhci_softc_t *);
Static void             uhci_rem_loop(uhci_softc_t *);

Static usbd_status      uhci_setup_isoc(struct usbd_pipe *);

Static struct usbd_xfer *
                       uhci_allocx(struct usbd_bus *, unsigned int);
Static void             uhci_freex(struct usbd_bus *, struct usbd_xfer *);
Static bool             uhci_dying(struct usbd_bus *);
Static void             uhci_get_lock(struct usbd_bus *, kmutex_t **);
Static int              uhci_roothub_ctrl(struct usbd_bus *,
                           usb_device_request_t *, void *, int);

Static int              uhci_device_ctrl_init(struct usbd_xfer *);
Static void             uhci_device_ctrl_fini(struct usbd_xfer *);
Static usbd_status      uhci_device_ctrl_transfer(struct usbd_xfer *);
Static usbd_status      uhci_device_ctrl_start(struct usbd_xfer *);
Static void             uhci_device_ctrl_abort(struct usbd_xfer *);
Static void             uhci_device_ctrl_close(struct usbd_pipe *);
Static void             uhci_device_ctrl_done(struct usbd_xfer *);

Static int              uhci_device_intr_init(struct usbd_xfer *);
Static void             uhci_device_intr_fini(struct usbd_xfer *);
Static usbd_status      uhci_device_intr_transfer(struct usbd_xfer *);
Static usbd_status      uhci_device_intr_start(struct usbd_xfer *);
Static void             uhci_device_intr_abort(struct usbd_xfer *);
Static void             uhci_device_intr_close(struct usbd_pipe *);
Static void             uhci_device_intr_done(struct usbd_xfer *);

Static int              uhci_device_bulk_init(struct usbd_xfer *);
Static void             uhci_device_bulk_fini(struct usbd_xfer *);
Static usbd_status      uhci_device_bulk_transfer(struct usbd_xfer *);
Static usbd_status      uhci_device_bulk_start(struct usbd_xfer *);
Static void             uhci_device_bulk_abort(struct usbd_xfer *);
Static void             uhci_device_bulk_close(struct usbd_pipe *);
Static void             uhci_device_bulk_done(struct usbd_xfer *);

Static int              uhci_device_isoc_init(struct usbd_xfer *);
Static void             uhci_device_isoc_fini(struct usbd_xfer *);
Static usbd_status      uhci_device_isoc_transfer(struct usbd_xfer *);
Static void             uhci_device_isoc_abort(struct usbd_xfer *);
Static void             uhci_device_isoc_close(struct usbd_pipe *);
Static void             uhci_device_isoc_done(struct usbd_xfer *);

Static usbd_status      uhci_root_intr_transfer(struct usbd_xfer *);
Static usbd_status      uhci_root_intr_start(struct usbd_xfer *);
Static void             uhci_root_intr_abort(struct usbd_xfer *);
Static void             uhci_root_intr_close(struct usbd_pipe *);
Static void             uhci_root_intr_done(struct usbd_xfer *);

Static usbd_status      uhci_open(struct usbd_pipe *);
Static void             uhci_poll(struct usbd_bus *);
Static void             uhci_softintr(void *);

Static void             uhci_add_intr(uhci_softc_t *, uhci_soft_qh_t *);
Static void             uhci_remove_intr(uhci_softc_t *, uhci_soft_qh_t *);
Static usbd_status      uhci_device_setintr(uhci_softc_t *,
                           struct uhci_pipe *, int);

Static void             uhci_device_clear_toggle(struct usbd_pipe *);
Static void             uhci_noop(struct usbd_pipe *);

static inline uhci_soft_qh_t *
                       uhci_find_prev_qh(uhci_soft_qh_t *, uhci_soft_qh_t *);

#ifdef UHCI_DEBUG
Static void             uhci_dump_all(uhci_softc_t *);
Static void             uhci_dumpregs(uhci_softc_t *);
Static void             uhci_dump_qhs(uhci_soft_qh_t *);
Static void             uhci_dump_qh(uhci_soft_qh_t *);
Static void             uhci_dump_tds(uhci_soft_td_t *);
Static void             uhci_dump_td(uhci_soft_td_t *);
Static void             uhci_dump_ii(struct uhci_xfer *);
void                    uhci_dump(void);
#endif

#define UBARR(sc) bus_space_barrier((sc)->iot, (sc)->ioh, 0, (sc)->sc_size, \
                       BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
#define UWRITE1(sc, r, x) \
do { UBARR(sc); bus_space_write_1((sc)->iot, (sc)->ioh, (r), (x)); \
} while (/*CONSTCOND*/0)
#define UWRITE2(sc, r, x) \
do { UBARR(sc); bus_space_write_2((sc)->iot, (sc)->ioh, (r), (x)); \
} while (/*CONSTCOND*/0)
#define UWRITE4(sc, r, x) \
do { UBARR(sc); bus_space_write_4((sc)->iot, (sc)->ioh, (r), (x)); \
} while (/*CONSTCOND*/0)

static __inline uint8_t
UREAD1(uhci_softc_t *sc, bus_size_t r)
{

       UBARR(sc);
       return bus_space_read_1(sc->iot, sc->ioh, r);
}

static __inline uint16_t
UREAD2(uhci_softc_t *sc, bus_size_t r)
{

       UBARR(sc);
       return bus_space_read_2(sc->iot, sc->ioh, r);
}

#ifdef UHCI_DEBUG
static __inline uint32_t
UREAD4(uhci_softc_t *sc, bus_size_t r)
{

       UBARR(sc);
       return bus_space_read_4(sc->iot, sc->ioh, r);
}
#endif

#define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
#define UHCISTS(sc) UREAD2(sc, UHCI_STS)

#define UHCI_RESET_TIMEOUT 100  /* ms, reset timeout */

#define UHCI_CURFRAME(sc) (UREAD2(sc, UHCI_FRNUM) & UHCI_FRNUM_MASK)

const struct usbd_bus_methods uhci_bus_methods = {
       .ubm_open =     uhci_open,
       .ubm_softint =  uhci_softintr,
       .ubm_dopoll =   uhci_poll,
       .ubm_allocx =   uhci_allocx,
       .ubm_freex =    uhci_freex,
       .ubm_abortx =   uhci_abortx,
       .ubm_dying =    uhci_dying,
       .ubm_getlock =  uhci_get_lock,
       .ubm_rhctrl =   uhci_roothub_ctrl,
};

const struct usbd_pipe_methods uhci_root_intr_methods = {
       .upm_transfer = uhci_root_intr_transfer,
       .upm_start =    uhci_root_intr_start,
       .upm_abort =    uhci_root_intr_abort,
       .upm_close =    uhci_root_intr_close,
       .upm_cleartoggle =      uhci_noop,
       .upm_done =     uhci_root_intr_done,
};

const struct usbd_pipe_methods uhci_device_ctrl_methods = {
       .upm_init =     uhci_device_ctrl_init,
       .upm_fini =     uhci_device_ctrl_fini,
       .upm_transfer = uhci_device_ctrl_transfer,
       .upm_start =    uhci_device_ctrl_start,
       .upm_abort =    uhci_device_ctrl_abort,
       .upm_close =    uhci_device_ctrl_close,
       .upm_cleartoggle =      uhci_noop,
       .upm_done =     uhci_device_ctrl_done,
};

const struct usbd_pipe_methods uhci_device_intr_methods = {
       .upm_init =     uhci_device_intr_init,
       .upm_fini =     uhci_device_intr_fini,
       .upm_transfer = uhci_device_intr_transfer,
       .upm_start =    uhci_device_intr_start,
       .upm_abort =    uhci_device_intr_abort,
       .upm_close =    uhci_device_intr_close,
       .upm_cleartoggle =      uhci_device_clear_toggle,
       .upm_done =     uhci_device_intr_done,
};

const struct usbd_pipe_methods uhci_device_bulk_methods = {
       .upm_init =     uhci_device_bulk_init,
       .upm_fini =     uhci_device_bulk_fini,
       .upm_transfer = uhci_device_bulk_transfer,
       .upm_start =    uhci_device_bulk_start,
       .upm_abort =    uhci_device_bulk_abort,
       .upm_close =    uhci_device_bulk_close,
       .upm_cleartoggle =      uhci_device_clear_toggle,
       .upm_done =     uhci_device_bulk_done,
};

const struct usbd_pipe_methods uhci_device_isoc_methods = {
       .upm_init =     uhci_device_isoc_init,
       .upm_fini =     uhci_device_isoc_fini,
       .upm_transfer = uhci_device_isoc_transfer,
       .upm_abort =    uhci_device_isoc_abort,
       .upm_close =    uhci_device_isoc_close,
       .upm_cleartoggle =      uhci_noop,
       .upm_done =     uhci_device_isoc_done,
};

static inline void
uhci_add_intr_list(uhci_softc_t *sc, struct uhci_xfer *ux)
{

       TAILQ_INSERT_TAIL(&sc->sc_intrhead, ux, ux_list);
}

static inline void
uhci_del_intr_list(uhci_softc_t *sc, struct uhci_xfer *ux)
{

       TAILQ_REMOVE(&sc->sc_intrhead, ux, ux_list);
}

static inline uhci_soft_qh_t *
uhci_find_prev_qh(uhci_soft_qh_t *pqh, uhci_soft_qh_t *sqh)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(15, "pqh=%#jx sqh=%#jx", (uintptr_t)pqh, (uintptr_t)sqh, 0, 0);

       for (; pqh->hlink != sqh; pqh = pqh->hlink) {
#if defined(DIAGNOSTIC) || defined(UHCI_DEBUG)
               usb_syncmem(&pqh->dma,
                   pqh->offs + offsetof(uhci_qh_t, qh_hlink),
                   sizeof(pqh->qh.qh_hlink),
                   BUS_DMASYNC_POSTWRITE);
               if (le32toh(pqh->qh.qh_hlink) & UHCI_PTR_T) {
                       printf("%s: QH not found\n", __func__);
                       return NULL;
               }
#endif
       }
       return pqh;
}

void
uhci_globalreset(uhci_softc_t *sc)
{
       UHCICMD(sc, UHCI_CMD_GRESET);   /* global reset */
       usb_delay_ms(&sc->sc_bus, USB_BUS_RESET_DELAY); /* wait a little */
       UHCICMD(sc, 0);                 /* do nothing */
}

int
uhci_init(uhci_softc_t *sc)
{
       int i, j;
       uhci_soft_qh_t *clsqh, *chsqh, *bsqh, *sqh, *lsqh;
       uhci_soft_td_t *std;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

#ifdef UHCI_DEBUG
       thesc = sc;

       if (uhcidebug >= 2)
               uhci_dumpregs(sc);
#endif

       sc->sc_suspend = PWR_RESUME;

       UWRITE2(sc, UHCI_INTR, 0);              /* disable interrupts */
       uhci_globalreset(sc);                   /* reset the controller */
       uhci_reset(sc);

       /* Allocate and initialize real frame array. */
       int err = usb_allocmem(sc->sc_bus.ub_dmatag,
           UHCI_FRAMELIST_COUNT * sizeof(uhci_physaddr_t),
           UHCI_FRAMELIST_ALIGN, USBMALLOC_COHERENT, &sc->sc_dma);
       if (err)
               return err;
       sc->sc_pframes = KERNADDR(&sc->sc_dma, 0);
       /* set frame number to 0 */
       UWRITE2(sc, UHCI_FRNUM, 0);
       /* set frame list */
       UWRITE4(sc, UHCI_FLBASEADDR, DMAADDR(&sc->sc_dma, 0));

       /* Initialise mutex early for uhci_alloc_* */
       mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
       mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);

       /*
        * Allocate a TD, inactive, that hangs from the last QH.
        * This is to avoid a bug in the PIIX that makes it run berserk
        * otherwise.
        */
       std = uhci_alloc_std(sc);
       if (std == NULL)
               return ENOMEM;
       std->link.std = NULL;
       std->td.td_link = htole32(UHCI_PTR_T);
       std->td.td_status = htole32(0); /* inactive */
       std->td.td_token = htole32(0);
       std->td.td_buffer = htole32(0);
       usb_syncmem(&std->dma, std->offs, sizeof(std->td),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       /* Allocate the dummy QH marking the end and used for looping the QHs.*/
       lsqh = uhci_alloc_sqh(sc);
       if (lsqh == NULL)
               goto fail1;
       lsqh->hlink = NULL;
       lsqh->qh.qh_hlink = htole32(UHCI_PTR_T);        /* end of QH chain */
       lsqh->elink = std;
       lsqh->qh.qh_elink = htole32(std->physaddr | UHCI_PTR_TD);
       sc->sc_last_qh = lsqh;
       usb_syncmem(&lsqh->dma, lsqh->offs, sizeof(lsqh->qh),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       /* Allocate the dummy QH where bulk traffic will be queued. */
       bsqh = uhci_alloc_sqh(sc);
       if (bsqh == NULL)
               goto fail2;
       bsqh->hlink = lsqh;
       bsqh->qh.qh_hlink = htole32(lsqh->physaddr | UHCI_PTR_QH);
       bsqh->elink = NULL;
       bsqh->qh.qh_elink = htole32(UHCI_PTR_T);
       sc->sc_bulk_start = sc->sc_bulk_end = bsqh;
       usb_syncmem(&bsqh->dma, bsqh->offs, sizeof(bsqh->qh),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       /* Allocate dummy QH where high speed control traffic will be queued. */
       chsqh = uhci_alloc_sqh(sc);
       if (chsqh == NULL)
               goto fail3;
       chsqh->hlink = bsqh;
       chsqh->qh.qh_hlink = htole32(bsqh->physaddr | UHCI_PTR_QH);
       chsqh->elink = NULL;
       chsqh->qh.qh_elink = htole32(UHCI_PTR_T);
       sc->sc_hctl_start = sc->sc_hctl_end = chsqh;
       usb_syncmem(&chsqh->dma, chsqh->offs, sizeof(chsqh->qh),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       /* Allocate dummy QH where control traffic will be queued. */
       clsqh = uhci_alloc_sqh(sc);
       if (clsqh == NULL)
               goto fail4;
       clsqh->hlink = chsqh;
       clsqh->qh.qh_hlink = htole32(chsqh->physaddr | UHCI_PTR_QH);
       clsqh->elink = NULL;
       clsqh->qh.qh_elink = htole32(UHCI_PTR_T);
       sc->sc_lctl_start = sc->sc_lctl_end = clsqh;
       usb_syncmem(&clsqh->dma, clsqh->offs, sizeof(clsqh->qh),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       /*
        * Make all (virtual) frame list pointers point to the interrupt
        * queue heads and the interrupt queue heads at the control
        * queue head and point the physical frame list to the virtual.
        */
       for (i = 0; i < UHCI_VFRAMELIST_COUNT; i++) {
               std = uhci_alloc_std(sc);
               sqh = uhci_alloc_sqh(sc);
               if (std == NULL || sqh == NULL)
                       return USBD_NOMEM;
               std->link.sqh = sqh;
               std->td.td_link = htole32(sqh->physaddr | UHCI_PTR_QH);
               std->td.td_status = htole32(UHCI_TD_IOS); /* iso, inactive */
               std->td.td_token = htole32(0);
               std->td.td_buffer = htole32(0);
               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               sqh->hlink = clsqh;
               sqh->qh.qh_hlink = htole32(clsqh->physaddr | UHCI_PTR_QH);
               sqh->elink = NULL;
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               sc->sc_vframes[i].htd = std;
               sc->sc_vframes[i].etd = std;
               sc->sc_vframes[i].hqh = sqh;
               sc->sc_vframes[i].eqh = sqh;
               for (j = i;
                    j < UHCI_FRAMELIST_COUNT;
                    j += UHCI_VFRAMELIST_COUNT)
                       sc->sc_pframes[j] = htole32(std->physaddr);
       }
       usb_syncmem(&sc->sc_dma, 0,
           UHCI_FRAMELIST_COUNT * sizeof(uhci_physaddr_t),
           BUS_DMASYNC_PREWRITE);


       TAILQ_INIT(&sc->sc_intrhead);

       sc->sc_xferpool = pool_cache_init(sizeof(struct uhci_xfer), 0, 0, 0,
           "uhcixfer", NULL, IPL_USB, NULL, NULL, NULL);

       callout_init(&sc->sc_poll_handle, CALLOUT_MPSAFE);
       callout_setfunc(&sc->sc_poll_handle, uhci_poll_hub, sc);

       /* Set up the bus struct. */
       sc->sc_bus.ub_methods = &uhci_bus_methods;
       sc->sc_bus.ub_pipesize = sizeof(struct uhci_pipe);
       sc->sc_bus.ub_usedma = true;
       sc->sc_bus.ub_dmaflags = USBMALLOC_MULTISEG;

       UHCICMD(sc, UHCI_CMD_MAXP); /* Assume 64 byte packets at frame end */

       DPRINTF("Enabling...", 0, 0, 0, 0);

       err = uhci_run(sc, 1);          /* and here we go... */
       UWRITE2(sc, UHCI_INTR, UHCI_INTR_TOCRCIE | UHCI_INTR_RIE |
               UHCI_INTR_IOCE | UHCI_INTR_SPIE);       /* enable interrupts */
       return err;

fail4:
       uhci_free_sqh(sc, chsqh);
fail3:
       uhci_free_sqh(sc, lsqh);
fail2:
       uhci_free_sqh(sc, lsqh);
fail1:
       uhci_free_std(sc, std);

       return ENOMEM;
}

int
uhci_activate(device_t self, enum devact act)
{
       struct uhci_softc *sc = device_private(self);

       switch (act) {
       case DVACT_DEACTIVATE:
               sc->sc_dying = 1;
               return 0;
       default:
               return EOPNOTSUPP;
       }
}

void
uhci_childdet(device_t self, device_t child)
{
       struct uhci_softc *sc = device_private(self);

       KASSERT(sc->sc_child == child);
       sc->sc_child = NULL;
}

int
uhci_detach(struct uhci_softc *sc, int flags)
{
       int rv = 0;

       if (sc->sc_child != NULL)
               rv = config_detach(sc->sc_child, flags);

       if (rv != 0)
               return rv;

       KASSERT(sc->sc_intr_xfer == NULL);
       callout_halt(&sc->sc_poll_handle, NULL);
       callout_destroy(&sc->sc_poll_handle);

       mutex_destroy(&sc->sc_lock);
       mutex_destroy(&sc->sc_intr_lock);

       pool_cache_destroy(sc->sc_xferpool);

       /* XXX free other data structures XXX */

       return rv;
}

struct usbd_xfer *
uhci_allocx(struct usbd_bus *bus, unsigned int nframes)
{
       struct uhci_softc *sc = UHCI_BUS2SC(bus);
       struct usbd_xfer *xfer;

       xfer = pool_cache_get(sc->sc_xferpool, PR_WAITOK);
       if (xfer != NULL) {
               memset(xfer, 0, sizeof(struct uhci_xfer));

#ifdef DIAGNOSTIC
               struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
               uxfer->ux_isdone = true;
               xfer->ux_state = XFER_BUSY;
#endif
       }
       return xfer;
}

void
uhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
{
       struct uhci_softc *sc = UHCI_BUS2SC(bus);
       struct uhci_xfer *uxfer __diagused = UHCI_XFER2UXFER(xfer);

       KASSERTMSG(xfer->ux_state == XFER_BUSY ||
           xfer->ux_status == USBD_NOT_STARTED,
           "xfer %p state %d\n", xfer, xfer->ux_state);
       KASSERTMSG(uxfer->ux_isdone || xfer->ux_status == USBD_NOT_STARTED,
           "xfer %p not done\n", xfer);
#ifdef DIAGNOSTIC
       xfer->ux_state = XFER_FREE;
#endif
       pool_cache_put(sc->sc_xferpool, xfer);
}

Static bool
uhci_dying(struct usbd_bus *bus)
{
       struct uhci_softc *sc = UHCI_BUS2SC(bus);

       return sc->sc_dying;
}

Static void
uhci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
{
       struct uhci_softc *sc = UHCI_BUS2SC(bus);

       *lock = &sc->sc_lock;
}


/*
* Handle suspend/resume.
*
* We need to switch to polling mode here, because this routine is
* called from an interrupt context.  This is all right since we
* are almost suspended anyway.
*/
bool
uhci_resume(device_t dv, const pmf_qual_t *qual)
{
       uhci_softc_t *sc = device_private(dv);
       int cmd;

       cmd = UREAD2(sc, UHCI_CMD);
       UWRITE2(sc, UHCI_INTR, 0);
       uhci_globalreset(sc);
       uhci_reset(sc);
       if (cmd & UHCI_CMD_RS)
               uhci_run(sc, 0);

       /* restore saved state */
       UWRITE4(sc, UHCI_FLBASEADDR, DMAADDR(&sc->sc_dma, 0));
       UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
       UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);

       UHCICMD(sc, cmd | UHCI_CMD_FGR); /* force resume */
       usb_delay_ms(&sc->sc_bus, USB_RESUME_DELAY);
       UHCICMD(sc, cmd & ~UHCI_CMD_EGSM); /* back to normal */
       UWRITE2(sc, UHCI_INTR, UHCI_INTR_TOCRCIE |
           UHCI_INTR_RIE | UHCI_INTR_IOCE | UHCI_INTR_SPIE);
       UHCICMD(sc, UHCI_CMD_MAXP);
       uhci_run(sc, 1); /* and start traffic again */
       usb_delay_ms(&sc->sc_bus, USB_RESUME_RECOVERY);
#ifdef UHCI_DEBUG
       if (uhcidebug >= 2)
               uhci_dumpregs(sc);
#endif

       mutex_enter(&sc->sc_lock);
       sc->sc_suspend = PWR_RESUME;
       if (sc->sc_intr_xfer != NULL)
               callout_schedule(&sc->sc_poll_handle, sc->sc_ival);
       mutex_exit(&sc->sc_lock);

       return true;
}

bool
uhci_suspend(device_t dv, const pmf_qual_t *qual)
{
       uhci_softc_t *sc = device_private(dv);
       int cmd;

       mutex_enter(&sc->sc_lock);
       sc->sc_suspend = PWR_SUSPEND;
       if (sc->sc_intr_xfer != NULL)
               callout_halt(&sc->sc_poll_handle, &sc->sc_lock);
       mutex_exit(&sc->sc_lock);

       cmd = UREAD2(sc, UHCI_CMD);

#ifdef UHCI_DEBUG
       if (uhcidebug >= 2)
               uhci_dumpregs(sc);
#endif

       uhci_run(sc, 0); /* stop the controller */
       cmd &= ~UHCI_CMD_RS;

       /* save some state if BIOS doesn't */
       sc->sc_saved_frnum = UREAD2(sc, UHCI_FRNUM);
       sc->sc_saved_sof = UREAD1(sc, UHCI_SOF);

       UWRITE2(sc, UHCI_INTR, 0); /* disable intrs */

       UHCICMD(sc, cmd | UHCI_CMD_EGSM); /* enter suspend */
       usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);

       return true;
}

#ifdef UHCI_DEBUG
Static void
uhci_dumpregs(uhci_softc_t *sc)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTF("cmd =%04jx  sts    =%04jx  intr   =%04jx  frnum =%04jx",
           UREAD2(sc, UHCI_CMD), UREAD2(sc, UHCI_STS),
           UREAD2(sc, UHCI_INTR), UREAD2(sc, UHCI_FRNUM));
       DPRINTF("sof =%04jx  portsc1=%04jx  portsc2=%04jx  flbase=%08jx",
           UREAD1(sc, UHCI_SOF), UREAD2(sc, UHCI_PORTSC1),
           UREAD2(sc, UHCI_PORTSC2), UREAD4(sc, UHCI_FLBASEADDR));
}

void
uhci_dump_td(uhci_soft_td_t *p)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       usb_syncmem(&p->dma, p->offs, sizeof(p->td),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

       DPRINTF("TD(%#jx) at 0x%08jx", (uintptr_t)p, p->physaddr, 0, 0);
       DPRINTF("   link=0x%08jx status=0x%08jx "
           "token=0x%08x buffer=0x%08x",
            le32toh(p->td.td_link),
            le32toh(p->td.td_status),
            le32toh(p->td.td_token),
            le32toh(p->td.td_buffer));

       DPRINTF("bitstuff=%jd crcto   =%jd nak     =%jd babble  =%jd",
           !!(le32toh(p->td.td_status) & UHCI_TD_BITSTUFF),
           !!(le32toh(p->td.td_status) & UHCI_TD_CRCTO),
           !!(le32toh(p->td.td_status) & UHCI_TD_NAK),
           !!(le32toh(p->td.td_status) & UHCI_TD_BABBLE));
       DPRINTF("dbuffer =%jd stalled =%jd active  =%jd ioc     =%jd",
           !!(le32toh(p->td.td_status) & UHCI_TD_DBUFFER),
           !!(le32toh(p->td.td_status) & UHCI_TD_STALLED),
           !!(le32toh(p->td.td_status) & UHCI_TD_ACTIVE),
           !!(le32toh(p->td.td_status) & UHCI_TD_IOC));
       DPRINTF("ios     =%jd ls      =%jd spd     =%jd",
           !!(le32toh(p->td.td_status) & UHCI_TD_IOS),
           !!(le32toh(p->td.td_status) & UHCI_TD_LS),
           !!(le32toh(p->td.td_status) & UHCI_TD_SPD), 0);
       DPRINTF("errcnt  =%jd actlen  =%jd pid=%02jx",
           UHCI_TD_GET_ERRCNT(le32toh(p->td.td_status)),
           UHCI_TD_GET_ACTLEN(le32toh(p->td.td_status)),
           UHCI_TD_GET_PID(le32toh(p->td.td_token)), 0);
       DPRINTF("addr=%jd  endpt=%jd  D=%jd  maxlen=%jd,",
           UHCI_TD_GET_DEVADDR(le32toh(p->td.td_token)),
           UHCI_TD_GET_ENDPT(le32toh(p->td.td_token)),
           UHCI_TD_GET_DT(le32toh(p->td.td_token)),
           UHCI_TD_GET_MAXLEN(le32toh(p->td.td_token)));
}

void
uhci_dump_qh(uhci_soft_qh_t *sqh)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

       DPRINTF("QH(%#jx) at 0x%08jx: hlink=%08jx elink=%08jx", (uintptr_t)sqh,
           (int)sqh->physaddr, le32toh(sqh->qh.qh_hlink),
           le32toh(sqh->qh.qh_elink));

       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh), BUS_DMASYNC_PREREAD);
}


#if 1
void
uhci_dump(void)
{
       uhci_dump_all(thesc);
}
#endif

void
uhci_dump_all(uhci_softc_t *sc)
{
       uhci_dumpregs(sc);
       /*printf("framelist[i].link = %08x\n", sc->sc_framelist[0].link);*/
       uhci_dump_qhs(sc->sc_lctl_start);
}


void
uhci_dump_qhs(uhci_soft_qh_t *sqh)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       uhci_dump_qh(sqh);

       /*
        * uhci_dump_qhs displays all the QHs and TDs from the given QH onwards
        * Traverses sideways first, then down.
        *
        * QH1
        * QH2
        * No QH
        * TD2.1
        * TD2.2
        * TD1.1
        * etc.
        *
        * TD2.x being the TDs queued at QH2 and QH1 being referenced from QH1.
        */

       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       if (sqh->hlink != NULL && !(le32toh(sqh->qh.qh_hlink) & UHCI_PTR_T))
               uhci_dump_qhs(sqh->hlink);
       else
               DPRINTF("No QH", 0, 0, 0, 0);
       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh), BUS_DMASYNC_PREREAD);

       if (sqh->elink != NULL && !(le32toh(sqh->qh.qh_elink) & UHCI_PTR_T))
               uhci_dump_tds(sqh->elink);
       else
               DPRINTF("No QH", 0, 0, 0, 0);
}

void
uhci_dump_tds(uhci_soft_td_t *std)
{
       uhci_soft_td_t *td;
       int stop;

       for (td = std; td != NULL; td = td->link.std) {
               uhci_dump_td(td);

               /*
                * Check whether the link pointer in this TD marks
                * the link pointer as end of queue. This avoids
                * printing the free list in case the queue/TD has
                * already been moved there (seatbelt).
                */
               usb_syncmem(&td->dma, td->offs + offsetof(uhci_td_t, td_link),
                   sizeof(td->td.td_link),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               stop = (le32toh(td->td.td_link) & UHCI_PTR_T ||
                       le32toh(td->td.td_link) == 0);
               usb_syncmem(&td->dma, td->offs + offsetof(uhci_td_t, td_link),
                   sizeof(td->td.td_link), BUS_DMASYNC_PREREAD);
               if (stop)
                       break;
       }
}

Static void
uhci_dump_ii(struct uhci_xfer *ux)
{
       struct usbd_pipe *pipe;
       usb_endpoint_descriptor_t *ed;
       struct usbd_device *dev;

       if (ux == NULL) {
               printf("ux NULL\n");
               return;
       }
       pipe = ux->ux_xfer.ux_pipe;
       if (pipe == NULL) {
               printf("ux %p: done=%d pipe=NULL\n", ux, ux->ux_isdone);
               return;
       }
       if (pipe->up_endpoint == NULL) {
               printf("ux %p: done=%d pipe=%p pipe->up_endpoint=NULL\n",
                      ux, ux->ux_isdone, pipe);
               return;
       }
       if (pipe->up_dev == NULL) {
               printf("ux %p: done=%d pipe=%p pipe->up_dev=NULL\n",
                      ux, ux->ux_isdone, pipe);
               return;
       }
       ed = pipe->up_endpoint->ue_edesc;
       dev = pipe->up_dev;
       printf("ux %p: done=%d dev=%p vid=0x%04x pid=0x%04x addr=%d pipe=%p ep=0x%02x attr=0x%02x\n",
              ux, ux->ux_isdone, dev,
              UGETW(dev->ud_ddesc.idVendor),
              UGETW(dev->ud_ddesc.idProduct),
              dev->ud_addr, pipe,
              ed->bEndpointAddress, ed->bmAttributes);
}

void uhci_dump_iis(struct uhci_softc *sc);
void
uhci_dump_iis(struct uhci_softc *sc)
{
       struct uhci_xfer *ux;

       printf("interrupt list:\n");
       TAILQ_FOREACH(ux, &sc->sc_intrhead, ux_list)
               uhci_dump_ii(ux);
}

void iidump(void);
void iidump(void) { uhci_dump_iis(thesc); }

#endif

/*
* This routine is executed periodically and simulates interrupts
* from the root controller interrupt pipe for port status change.
*/
void
uhci_poll_hub(void *addr)
{
       struct uhci_softc *sc = addr;
       struct usbd_xfer *xfer;
       u_char *p;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       mutex_enter(&sc->sc_lock);

       /*
        * If the intr xfer has completed or been synchronously
        * aborted, we have nothing to do.
        */
       xfer = sc->sc_intr_xfer;
       if (xfer == NULL)
               goto out;
       KASSERT(xfer->ux_status == USBD_IN_PROGRESS);

       /*
        * If the intr xfer for which we were scheduled is done, and
        * another intr xfer has been submitted, let that one be dealt
        * with when the callout fires again.
        *
        * The call to callout_pending is racy, but the transition
        * from pending to invoking happens atomically.  The
        * callout_ack ensures callout_invoking does not return true
        * due to this invocation of the callout; the lock ensures the
        * next invocation of the callout cannot callout_ack (unless it
        * had already run to completion and nulled sc->sc_intr_xfer,
        * in which case would have bailed out already).
        */
       callout_ack(&sc->sc_poll_handle);
       if (callout_pending(&sc->sc_poll_handle) ||
           callout_invoking(&sc->sc_poll_handle))
               goto out;

       /*
        * Check flags for the two interrupt ports, and set them in the
        * buffer if an interrupt arrived; otherwise arrange .
        */
       p = xfer->ux_buf;
       p[0] = 0;
       if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC|UHCI_PORTSC_OCIC))
               p[0] |= 1<<1;
       if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC|UHCI_PORTSC_OCIC))
               p[0] |= 1<<2;
       if (p[0] == 0) {
               /*
                * No change -- try again in a while, unless we're
                * suspending, in which case we'll try again after
                * resume.
                */
               if (sc->sc_suspend != PWR_SUSPEND)
                       callout_schedule(&sc->sc_poll_handle, sc->sc_ival);
               goto out;
       }

       /*
        * Interrupt completed, and the xfer has not been completed or
        * synchronously aborted.  Complete the xfer now.
        */
       xfer->ux_actlen = 1;
       xfer->ux_status = USBD_NORMAL_COMPLETION;
#ifdef DIAGNOSTIC
       UHCI_XFER2UXFER(xfer)->ux_isdone = true;
#endif
       usb_transfer_complete(xfer);

out:    mutex_exit(&sc->sc_lock);
}

void
uhci_root_intr_done(struct usbd_xfer *xfer)
{
       struct uhci_softc *sc = UHCI_XFER2SC(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));

       /* Claim the xfer so it doesn't get completed again.  */
       KASSERT(sc->sc_intr_xfer == xfer);
       KASSERT(xfer->ux_status != USBD_IN_PROGRESS);
       sc->sc_intr_xfer = NULL;
}

/*
* Let the last QH loop back to the high speed control transfer QH.
* This is what intel calls "bandwidth reclamation" and improves
* USB performance a lot for some devices.
* If we are already looping, just count it.
*/
void
uhci_add_loop(uhci_softc_t *sc)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

#ifdef UHCI_DEBUG
       if (uhcinoloop)
               return;
#endif
       if (++sc->sc_loops == 1) {
               DPRINTFN(5, "add loop", 0, 0, 0, 0);
               /* Note, we don't loop back the soft pointer. */
               sc->sc_last_qh->qh.qh_hlink =
                   htole32(sc->sc_hctl_start->physaddr | UHCI_PTR_QH);
               usb_syncmem(&sc->sc_last_qh->dma,
                   sc->sc_last_qh->offs + offsetof(uhci_qh_t, qh_hlink),
                   sizeof(sc->sc_last_qh->qh.qh_hlink),
                   BUS_DMASYNC_PREWRITE);
       }
}

void
uhci_rem_loop(uhci_softc_t *sc)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

#ifdef UHCI_DEBUG
       if (uhcinoloop)
               return;
#endif
       if (--sc->sc_loops == 0) {
               DPRINTFN(5, "remove loop", 0, 0, 0, 0);
               sc->sc_last_qh->qh.qh_hlink = htole32(UHCI_PTR_T);
               usb_syncmem(&sc->sc_last_qh->dma,
                   sc->sc_last_qh->offs + offsetof(uhci_qh_t, qh_hlink),
                   sizeof(sc->sc_last_qh->qh.qh_hlink),
                   BUS_DMASYNC_PREWRITE);
       }
}

/* Add high speed control QH, called with lock held. */
void
uhci_add_hs_ctrl(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *eqh;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       KASSERT(mutex_owned(&sc->sc_lock));

       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);
       eqh = sc->sc_hctl_end;
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink),
           BUS_DMASYNC_POSTWRITE);
       sqh->hlink       = eqh->hlink;
       sqh->qh.qh_hlink = eqh->qh.qh_hlink;
       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
           BUS_DMASYNC_PREWRITE);
       eqh->hlink       = sqh;
       eqh->qh.qh_hlink = htole32(sqh->physaddr | UHCI_PTR_QH);
       sc->sc_hctl_end = sqh;
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
#ifdef UHCI_CTL_LOOP
       uhci_add_loop(sc);
#endif
}

/* Remove high speed control QH, called with lock held. */
void
uhci_remove_hs_ctrl(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *pqh;
       uint32_t elink;

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);
#ifdef UHCI_CTL_LOOP
       uhci_rem_loop(sc);
#endif
       /*
        * The T bit should be set in the elink of the QH so that the HC
        * doesn't follow the pointer.  This condition may fail if the
        * the transferred packet was short so that the QH still points
        * at the last used TD.
        * In this case we set the T bit and wait a little for the HC
        * to stop looking at the TD.
        * Note that if the TD chain is large enough, the controller
        * may still be looking at the chain at the end of this function.
        * uhci_free_std_chain() will make sure the controller stops
        * looking at it quickly, but until then we should not change
        * sqh->hlink.
        */
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       elink = le32toh(sqh->qh.qh_elink);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink), BUS_DMASYNC_PREREAD);
       if (!(elink & UHCI_PTR_T)) {
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               delay(UHCI_QH_REMOVE_DELAY);
       }

       pqh = uhci_find_prev_qh(sc->sc_hctl_start, sqh);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(sqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       pqh->hlink = sqh->hlink;
       pqh->qh.qh_hlink = sqh->qh.qh_hlink;
       usb_syncmem(&pqh->dma, pqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(pqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       delay(UHCI_QH_REMOVE_DELAY);
       if (sc->sc_hctl_end == sqh)
               sc->sc_hctl_end = pqh;
}

/* Add low speed control QH, called with lock held. */
void
uhci_add_ls_ctrl(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *eqh;

       KASSERT(mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);

       eqh = sc->sc_lctl_end;
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       sqh->hlink = eqh->hlink;
       sqh->qh.qh_hlink = eqh->qh.qh_hlink;
       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
           BUS_DMASYNC_PREWRITE);
       eqh->hlink = sqh;
       eqh->qh.qh_hlink = htole32(sqh->physaddr | UHCI_PTR_QH);
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       sc->sc_lctl_end = sqh;
}

/* Remove low speed control QH, called with lock held. */
void
uhci_remove_ls_ctrl(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *pqh;
       uint32_t elink;

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);

       /* See comment in uhci_remove_hs_ctrl() */
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       elink = le32toh(sqh->qh.qh_elink);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink), BUS_DMASYNC_PREREAD);
       if (!(elink & UHCI_PTR_T)) {
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               delay(UHCI_QH_REMOVE_DELAY);
       }
       pqh = uhci_find_prev_qh(sc->sc_lctl_start, sqh);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(sqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       pqh->hlink = sqh->hlink;
       pqh->qh.qh_hlink = sqh->qh.qh_hlink;
       usb_syncmem(&pqh->dma, pqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(pqh->qh.qh_hlink),
           BUS_DMASYNC_PREWRITE);
       delay(UHCI_QH_REMOVE_DELAY);
       if (sc->sc_lctl_end == sqh)
               sc->sc_lctl_end = pqh;
}

/* Add bulk QH, called with lock held. */
void
uhci_add_bulk(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *eqh;

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);

       eqh = sc->sc_bulk_end;
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       sqh->hlink = eqh->hlink;
       sqh->qh.qh_hlink = eqh->qh.qh_hlink;
       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
           BUS_DMASYNC_PREWRITE);
       eqh->hlink = sqh;
       eqh->qh.qh_hlink = htole32(sqh->physaddr | UHCI_PTR_QH);
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       sc->sc_bulk_end = sqh;
       uhci_add_loop(sc);
}

/* Remove bulk QH, called with lock held. */
void
uhci_remove_bulk(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       uhci_soft_qh_t *pqh;

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(10, "sqh %#jx", (uintptr_t)sqh, 0, 0, 0);

       uhci_rem_loop(sc);
       /* See comment in uhci_remove_hs_ctrl() */
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       if (!(sqh->qh.qh_elink & htole32(UHCI_PTR_T))) {
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               delay(UHCI_QH_REMOVE_DELAY);
       }
       pqh = uhci_find_prev_qh(sc->sc_bulk_start, sqh);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(sqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       pqh->hlink       = sqh->hlink;
       pqh->qh.qh_hlink = sqh->qh.qh_hlink;
       usb_syncmem(&pqh->dma, pqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(pqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       delay(UHCI_QH_REMOVE_DELAY);
       if (sc->sc_bulk_end == sqh)
               sc->sc_bulk_end = pqh;
}

Static int uhci_intr1(uhci_softc_t *);

int
uhci_intr(void *arg)
{
       uhci_softc_t *sc = arg;
       int ret = 0;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       mutex_spin_enter(&sc->sc_intr_lock);

       if (sc->sc_dying || !device_has_power(sc->sc_dev))
               goto done;

       if (sc->sc_bus.ub_usepolling || UREAD2(sc, UHCI_INTR) == 0) {
               DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0);
               goto done;
       }

       ret = uhci_intr1(sc);

done:
       mutex_spin_exit(&sc->sc_intr_lock);
       return ret;
}

int
uhci_intr1(uhci_softc_t *sc)
{
       int status;
       int ack;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

#ifdef UHCI_DEBUG
       if (uhcidebug >= 15) {
               DPRINTF("sc %#jx", (uintptr_t)sc, 0, 0, 0);
               uhci_dumpregs(sc);
       }
#endif

       KASSERT(mutex_owned(&sc->sc_intr_lock));

       status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
       /* Check if the interrupt was for us. */
       if (status == 0)
               return 0;

       if (sc->sc_suspend != PWR_RESUME) {
#ifdef DIAGNOSTIC
               printf("%s: interrupt while not operating ignored\n",
                      device_xname(sc->sc_dev));
#endif
               UWRITE2(sc, UHCI_STS, status); /* acknowledge the ints */
               return 0;
       }

       ack = 0;
       if (status & UHCI_STS_USBINT)
               ack |= UHCI_STS_USBINT;
       if (status & UHCI_STS_USBEI)
               ack |= UHCI_STS_USBEI;
       if (status & UHCI_STS_RD) {
               ack |= UHCI_STS_RD;
#ifdef UHCI_DEBUG
               printf("%s: resume detect\n", device_xname(sc->sc_dev));
#endif
       }
       if (status & UHCI_STS_HSE) {
               ack |= UHCI_STS_HSE;
               printf("%s: host system error\n", device_xname(sc->sc_dev));
       }
       if (status & UHCI_STS_HCPE) {
               ack |= UHCI_STS_HCPE;
               printf("%s: host controller process error\n",
                      device_xname(sc->sc_dev));
       }

       /* When HCHalted=1 and Run/Stop=0 , it is normal */
       if ((status & UHCI_STS_HCH) && (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS)) {
               /* no acknowledge needed */
               if (!sc->sc_dying) {
                       printf("%s: host controller halted\n",
                           device_xname(sc->sc_dev));
#ifdef UHCI_DEBUG
                       uhci_dump_all(sc);
#endif
               }
               sc->sc_dying = 1;
       }

       if (!ack)
               return 0;       /* nothing to acknowledge */
       UWRITE2(sc, UHCI_STS, ack); /* acknowledge the ints */

       usb_schedsoftintr(&sc->sc_bus);

       DPRINTFN(15, "sc %#jx done", (uintptr_t)sc, 0, 0, 0);

       return 1;
}

void
uhci_softintr(void *v)
{
       struct usbd_bus *bus = v;
       uhci_softc_t *sc = UHCI_BUS2SC(bus);
       struct uhci_xfer *ux, *nextux;
       ux_completeq_t cq;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTF("sc %#jx", (uintptr_t)sc, 0, 0, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       TAILQ_INIT(&cq);
       /*
        * Interrupts on UHCI really suck.  When the host controller
        * interrupts because a transfer is completed there is no
        * way of knowing which transfer it was.  You can scan down
        * the TDs and QHs of the previous frame to limit the search,
        * but that assumes that the interrupt was not delayed by more
        * than 1 ms, which may not always be true (e.g. after debug
        * output on a slow console).
        * We scan all interrupt descriptors to see if any have
        * completed.
        */
       TAILQ_FOREACH_SAFE(ux, &sc->sc_intrhead, ux_list, nextux) {
               uhci_check_intr(sc, ux, &cq);
       }

       /*
        * We abuse ux_list for the interrupt and complete lists and
        * interrupt transfers will get re-added here so use
        * the _SAFE version of TAILQ_FOREACH.
        */
       TAILQ_FOREACH_SAFE(ux, &cq, ux_list, nextux) {
               DPRINTF("ux %#jx", (uintptr_t)ux, 0, 0, 0);
               usb_transfer_complete(&ux->ux_xfer);
       }

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
}

/* Check for an interrupt. */
void
uhci_check_intr(uhci_softc_t *sc, struct uhci_xfer *ux, ux_completeq_t *cqp)
{
       uhci_soft_td_t *std, *fstd = NULL, *lstd = NULL;
       uint32_t status;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(15, "ux %#jx", (uintptr_t)ux, 0, 0, 0);

       KASSERT(ux != NULL);

       struct usbd_xfer *xfer = &ux->ux_xfer;
       if (xfer->ux_status == USBD_CANCELLED ||
           xfer->ux_status == USBD_TIMEOUT) {
               DPRINTF("aborted xfer %#jx", (uintptr_t)xfer, 0, 0, 0);
               return;
       }

       switch (ux->ux_type) {
       case UX_CTRL:
               fstd = ux->ux_setup;
               lstd = ux->ux_stat;
               break;
       case UX_BULK:
       case UX_INTR:
       case UX_ISOC:
               fstd = ux->ux_stdstart;
               lstd = ux->ux_stdend;
               break;
       default:
               KASSERT(false);
               break;
       }
       if (fstd == NULL)
               return;

       KASSERT(lstd != NULL);

       usb_syncmem(&lstd->dma,
           lstd->offs + offsetof(uhci_td_t, td_status),
           sizeof(lstd->td.td_status),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       status = le32toh(lstd->td.td_status);
       usb_syncmem(&lstd->dma,
           lstd->offs + offsetof(uhci_td_t, td_status),
           sizeof(lstd->td.td_status),
           BUS_DMASYNC_PREREAD);

       /* If the last TD is not marked active we can complete */
       if (!(status & UHCI_TD_ACTIVE)) {
done:
               DPRINTFN(12, "ux=%#jx done", (uintptr_t)ux, 0, 0, 0);
               uhci_idone(ux, cqp);
               return;
       }

       /*
        * If the last TD is still active we need to check whether there
        * is an error somewhere in the middle, or whether there was a
        * short packet (SPD and not ACTIVE).
        */
       DPRINTFN(12, "active ux=%#jx", (uintptr_t)ux, 0, 0, 0);
       for (std = fstd; std != lstd; std = std->link.std) {
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               status = le32toh(std->td.td_status);
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status), BUS_DMASYNC_PREREAD);

               /* If there's an active TD the xfer isn't done. */
               if (status & UHCI_TD_ACTIVE) {
                       DPRINTFN(12, "ux=%#jx std=%#jx still active",
                           (uintptr_t)ux, (uintptr_t)std, 0, 0);
                       return;
               }

               /* Any kind of error makes the xfer done. */
               if (status & UHCI_TD_STALLED)
                       goto done;

               /*
                * If the data phase of a control transfer is short, we need
                * to complete the status stage
                */

               if ((status & UHCI_TD_SPD) && ux->ux_type == UX_CTRL) {
                       struct uhci_pipe *upipe =
                           UHCI_PIPE2UPIPE(xfer->ux_pipe);
                       uhci_soft_qh_t *sqh = upipe->ctrl.sqh;
                       uhci_soft_td_t *stat = upipe->ctrl.stat;

                       DPRINTFN(12, "ux=%#jx std=%#jx control status"
                           "phase needs completion", (uintptr_t)ux,
                           (uintptr_t)ux->ux_stdstart, 0, 0);

                       sqh->qh.qh_elink =
                           htole32(stat->physaddr | UHCI_PTR_TD);
                       usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
                           BUS_DMASYNC_PREWRITE);
                       break;
               }

               /* We want short packets, and it is short: it's done */
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_token),
                   sizeof(std->td.td_token),
                   BUS_DMASYNC_POSTWRITE);

               if ((status & UHCI_TD_SPD) &&
                       UHCI_TD_GET_ACTLEN(status) <
                       UHCI_TD_GET_MAXLEN(le32toh(std->td.td_token))) {
                       goto done;
               }
       }
}

/* Called with USB lock held. */
void
uhci_idone(struct uhci_xfer *ux, ux_completeq_t *cqp)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       struct usbd_xfer *xfer = &ux->ux_xfer;
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       uhci_soft_td_t *std;
       uint32_t status = 0, nstatus;
       int actlen;

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       DPRINTFN(12, "ux=%#jx", (uintptr_t)ux, 0, 0, 0);

       /*
        * Try to claim this xfer for completion.  If it has already
        * completed or aborted, drop it on the floor.
        */
       if (!usbd_xfer_trycomplete(xfer))
               return;

#ifdef DIAGNOSTIC
#ifdef UHCI_DEBUG
       if (ux->ux_isdone) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_ii(ux);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif
       KASSERT(!ux->ux_isdone);
       KASSERTMSG(!ux->ux_isdone, "xfer %p type %d status %d", xfer,
           ux->ux_type, xfer->ux_status);
       ux->ux_isdone = true;
#endif

       if (xfer->ux_nframes != 0) {
               /* Isoc transfer, do things differently. */
               uhci_soft_td_t **stds = upipe->isoc.stds;
               int i, n, nframes, len;

               DPRINTFN(5, "ux=%#jx isoc ready", (uintptr_t)ux, 0, 0, 0);

               nframes = xfer->ux_nframes;
               actlen = 0;
               n = ux->ux_curframe;
               for (i = 0; i < nframes; i++) {
                       std = stds[n];
#ifdef UHCI_DEBUG
                       if (uhcidebug >= 5) {
                               DPRINTF("isoc TD %jd", i, 0, 0, 0);
                               DPRINTF("--- dump start ---", 0, 0, 0, 0);
                               uhci_dump_td(std);
                               DPRINTF("--- dump end ---", 0, 0, 0, 0);
                       }
#endif
                       if (++n >= UHCI_VFRAMELIST_COUNT)
                               n = 0;
                       usb_syncmem(&std->dma,
                           std->offs + offsetof(uhci_td_t, td_status),
                           sizeof(std->td.td_status),
                           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
                       status = le32toh(std->td.td_status);
                       len = UHCI_TD_GET_ACTLEN(status);
                       xfer->ux_frlengths[i] = len;
                       actlen += len;
               }
               upipe->isoc.inuse -= nframes;
               xfer->ux_actlen = actlen;
               xfer->ux_status = USBD_NORMAL_COMPLETION;
               goto end;
       }

#ifdef UHCI_DEBUG
       DPRINTFN(10, "ux=%#jx, xfer=%#jx, pipe=%#jx ready", (uintptr_t)ux,
           (uintptr_t)xfer, (uintptr_t)upipe, 0);
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_tds(ux->ux_stdstart);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       /* The transfer is done, compute actual length and status. */
       actlen = 0;
       for (std = ux->ux_stdstart; std != NULL; std = std->link.std) {
               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               nstatus = le32toh(std->td.td_status);
               if (nstatus & UHCI_TD_ACTIVE)
                       break;

               status = nstatus;
               if (UHCI_TD_GET_PID(le32toh(std->td.td_token)) !=
                       UHCI_TD_PID_SETUP)
                       actlen += UHCI_TD_GET_ACTLEN(status);
               else {
                       /*
                        * UHCI will report CRCTO in addition to a STALL or NAK
                        * for a SETUP transaction.  See section 3.2.2, "TD
                        * CONTROL AND STATUS".
                        */
                       if (status & (UHCI_TD_STALLED | UHCI_TD_NAK))
                               status &= ~UHCI_TD_CRCTO;
               }
       }
       /* If there are left over TDs we need to update the toggle. */
       if (std != NULL)
               upipe->nexttoggle = UHCI_TD_GET_DT(le32toh(std->td.td_token));

       status &= UHCI_TD_ERROR;
       DPRINTFN(10, "actlen=%jd, status=%#jx", actlen, status, 0, 0);
       xfer->ux_actlen = actlen;
       if (status != 0) {

               DPRINTFN((status == UHCI_TD_STALLED) * 10,
                   "error, addr=%jd, endpt=0x%02jx",
                   xfer->ux_pipe->up_dev->ud_addr,
                   xfer->ux_pipe->up_endpoint->ue_edesc->bEndpointAddress,
                   0, 0);
               DPRINTFN((status == UHCI_TD_STALLED) * 10,
                   "bitstuff=%jd crcto   =%jd nak     =%jd babble  =%jd",
                   !!(status & UHCI_TD_BITSTUFF),
                   !!(status & UHCI_TD_CRCTO),
                   !!(status & UHCI_TD_NAK),
                   !!(status & UHCI_TD_BABBLE));
               DPRINTFN((status == UHCI_TD_STALLED) * 10,
                   "dbuffer =%jd stalled =%jd active  =%jd",
                   !!(status & UHCI_TD_DBUFFER),
                   !!(status & UHCI_TD_STALLED),
                   !!(status & UHCI_TD_ACTIVE),
                   0);

               if (status == UHCI_TD_STALLED)
                       xfer->ux_status = USBD_STALLED;
               else
                       xfer->ux_status = USBD_IOERROR; /* more info XXX */
       } else {
               xfer->ux_status = USBD_NORMAL_COMPLETION;
       }

end:
       uhci_del_intr_list(sc, ux);
       if (cqp)
               TAILQ_INSERT_TAIL(cqp, ux, ux_list);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
       DPRINTFN(12, "ux=%#jx done", (uintptr_t)ux, 0, 0, 0);
}

void
uhci_poll(struct usbd_bus *bus)
{
       uhci_softc_t *sc = UHCI_BUS2SC(bus);

       if (UREAD2(sc, UHCI_STS) & UHCI_STS_USBINT) {
               mutex_spin_enter(&sc->sc_intr_lock);
               uhci_intr1(sc);
               mutex_spin_exit(&sc->sc_intr_lock);
       }
}

void
uhci_reset(uhci_softc_t *sc)
{
       int n;

       UHCICMD(sc, UHCI_CMD_HCRESET);
       /* The reset bit goes low when the controller is done. */
       for (n = 0; n < UHCI_RESET_TIMEOUT &&
                   (UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET); n++)
               usb_delay_ms(&sc->sc_bus, 1);
       if (n >= UHCI_RESET_TIMEOUT)
               printf("%s: controller did not reset\n",
                      device_xname(sc->sc_dev));
}

usbd_status
uhci_run(uhci_softc_t *sc, int run)
{
       int n, running;
       uint16_t cmd;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       run = run != 0;

       DPRINTF("setting run=%jd", run, 0, 0, 0);
       cmd = UREAD2(sc, UHCI_CMD);
       if (run)
               cmd |= UHCI_CMD_RS;
       else
               cmd &= ~UHCI_CMD_RS;
       UHCICMD(sc, cmd);
       for (n = 0; n < 10; n++) {
               running = !(UREAD2(sc, UHCI_STS) & UHCI_STS_HCH);
               /* return when we've entered the state we want */
               if (run == running) {
                       DPRINTF("done cmd=%#jx sts=%#jx",
                           UREAD2(sc, UHCI_CMD), UREAD2(sc, UHCI_STS), 0, 0);
                       return USBD_NORMAL_COMPLETION;
               }
               usb_delay_ms(&sc->sc_bus, 1);
       }
       printf("%s: cannot %s\n", device_xname(sc->sc_dev),
              run ? "start" : "stop");
       return USBD_IOERROR;
}

/*
* Memory management routines.
*  uhci_alloc_std allocates TDs
*  uhci_alloc_sqh allocates QHs
* These two routines do their own free list management,
* partly for speed, partly because allocating DMAable memory
* has page size granularity so much memory would be wasted if
* only one TD/QH (32 bytes) was placed in each allocated chunk.
*/

uhci_soft_td_t *
uhci_alloc_std(uhci_softc_t *sc)
{
       uhci_soft_td_t *std;
       int i, offs;
       usb_dma_t dma;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       mutex_enter(&sc->sc_lock);
       if (sc->sc_freetds == NULL) {
               DPRINTFN(2, "allocating chunk", 0, 0, 0, 0);
               mutex_exit(&sc->sc_lock);

               int err = usb_allocmem(sc->sc_bus.ub_dmatag, UHCI_STD_SIZE * UHCI_STD_CHUNK,
                   UHCI_TD_ALIGN, USBMALLOC_COHERENT, &dma);
               if (err)
                       return NULL;

               mutex_enter(&sc->sc_lock);
               for (i = 0; i < UHCI_STD_CHUNK; i++) {
                       offs = i * UHCI_STD_SIZE;
                       std = KERNADDR(&dma, offs);
                       std->physaddr = DMAADDR(&dma, offs);
                       std->dma = dma;
                       std->offs = offs;
                       std->link.std = sc->sc_freetds;
                       sc->sc_freetds = std;
               }
       }
       std = sc->sc_freetds;
       sc->sc_freetds = std->link.std;
       mutex_exit(&sc->sc_lock);

       memset(&std->td, 0, sizeof(uhci_td_t));

       return std;
}

#define TD_IS_FREE 0x12345678

void
uhci_free_std_locked(uhci_softc_t *sc, uhci_soft_td_t *std)
{
       KASSERT(mutex_owned(&sc->sc_lock));

#ifdef DIAGNOSTIC
       if (le32toh(std->td.td_token) == TD_IS_FREE) {
               printf("%s: freeing free TD %p\n", __func__, std);
               return;
       }
       std->td.td_token = htole32(TD_IS_FREE);
#endif

       std->link.std = sc->sc_freetds;
       sc->sc_freetds = std;
}

void
uhci_free_std(uhci_softc_t *sc, uhci_soft_td_t *std)
{
       mutex_enter(&sc->sc_lock);
       uhci_free_std_locked(sc, std);
       mutex_exit(&sc->sc_lock);
}

uhci_soft_qh_t *
uhci_alloc_sqh(uhci_softc_t *sc)
{
       uhci_soft_qh_t *sqh;
       int i, offs;
       usb_dma_t dma;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       mutex_enter(&sc->sc_lock);
       if (sc->sc_freeqhs == NULL) {
               DPRINTFN(2, "allocating chunk", 0, 0, 0, 0);
               mutex_exit(&sc->sc_lock);

               int err = usb_allocmem(sc->sc_bus.ub_dmatag, UHCI_SQH_SIZE * UHCI_SQH_CHUNK,
                   UHCI_QH_ALIGN, USBMALLOC_COHERENT, &dma);
               if (err)
                       return NULL;

               mutex_enter(&sc->sc_lock);
               for (i = 0; i < UHCI_SQH_CHUNK; i++) {
                       offs = i * UHCI_SQH_SIZE;
                       sqh = KERNADDR(&dma, offs);
                       sqh->physaddr = DMAADDR(&dma, offs);
                       sqh->dma = dma;
                       sqh->offs = offs;
                       sqh->hlink = sc->sc_freeqhs;
                       sc->sc_freeqhs = sqh;
               }
       }
       sqh = sc->sc_freeqhs;
       sc->sc_freeqhs = sqh->hlink;
       mutex_exit(&sc->sc_lock);

       memset(&sqh->qh, 0, sizeof(uhci_qh_t));

       return sqh;
}

void
uhci_free_sqh(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       KASSERT(mutex_owned(&sc->sc_lock));

       sqh->hlink = sc->sc_freeqhs;
       sc->sc_freeqhs = sqh;
}

#if 0
void
uhci_free_std_chain(uhci_softc_t *sc, uhci_soft_td_t *std,
                   uhci_soft_td_t *stdend)
{
       uhci_soft_td_t *p;
       uint32_t td_link;

       /*
        * to avoid race condition with the controller which may be looking
        * at this chain, we need to first invalidate all links, and
        * then wait for the controller to move to another queue
        */
       for (p = std; p != stdend; p = p->link.std) {
               usb_syncmem(&p->dma,
                   p->offs + offsetof(uhci_td_t, td_link),
                   sizeof(p->td.td_link),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               td_link = le32toh(p->td.td_link);
               usb_syncmem(&p->dma,
                   p->offs + offsetof(uhci_td_t, td_link),
                   sizeof(p->td.td_link),
                   BUS_DMASYNC_PREREAD);
               if ((td_link & UHCI_PTR_T) == 0) {
                       p->td.td_link = htole32(UHCI_PTR_T);
                       usb_syncmem(&p->dma,
                           p->offs + offsetof(uhci_td_t, td_link),
                           sizeof(p->td.td_link),
                           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               }
       }
       delay(UHCI_QH_REMOVE_DELAY);

       for (; std != stdend; std = p) {
               p = std->link.std;
               uhci_free_std(sc, std);
       }
}
#endif

int
uhci_alloc_std_chain(uhci_softc_t *sc, struct usbd_xfer *xfer, int len,
   int rd, uhci_soft_td_t **sp)
{
       struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
       uint16_t flags = xfer->ux_flags;
       uhci_soft_td_t *p;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       DPRINTFN(8, "xfer=%#jx pipe=%#jx", (uintptr_t)xfer,
           (uintptr_t)xfer->ux_pipe, 0, 0);

       ASSERT_SLEEPABLE();
       KASSERT(sp);

       int maxp = UGETW(xfer->ux_pipe->up_endpoint->ue_edesc->wMaxPacketSize);
       if (maxp == 0) {
               printf("%s: maxp=0\n", __func__);
               return EINVAL;
       }
       size_t ntd = howmany(len, maxp);
       /*
        * if our transfer is bigger than PAGE_SIZE and maxp not a factor of
        * PAGE_SIZE then we will need another TD per page.
        */
       if (len > PAGE_SIZE && (PAGE_SIZE % maxp) != 0) {
               ntd += howmany(len, PAGE_SIZE);
       }

       /*
        * Might need one more TD if we're writing a ZLP
        */
       if (!rd && (flags & USBD_FORCE_SHORT_XFER)) {
               ntd++;
       }
       DPRINTFN(10, "maxp=%jd ntd=%jd", maxp, ntd, 0, 0);

       uxfer->ux_stds = NULL;
       uxfer->ux_nstd = ntd;
       if (ntd == 0) {
               *sp = NULL;
               DPRINTF("ntd=0", 0, 0, 0, 0);
               return 0;
       }
       uxfer->ux_stds = kmem_alloc(sizeof(uhci_soft_td_t *) * ntd,
           KM_SLEEP);

       for (int i = 0; i < ntd; i++) {
               p = uhci_alloc_std(sc);
               if (p == NULL) {
                       if (i != 0) {
                               uxfer->ux_nstd = i;
                               uhci_free_stds(sc, uxfer);
                       }
                       kmem_free(uxfer->ux_stds,
                           sizeof(uhci_soft_td_t *) * ntd);
                       return ENOMEM;
               }
               uxfer->ux_stds[i] = p;
       }

       *sp = uxfer->ux_stds[0];

       return 0;
}

Static void
uhci_free_stds(uhci_softc_t *sc, struct uhci_xfer *ux)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       DPRINTFN(8, "ux=%#jx", (uintptr_t)ux, 0, 0, 0);

       mutex_enter(&sc->sc_lock);
       for (size_t i = 0; i < ux->ux_nstd; i++) {
               uhci_soft_td_t *std = ux->ux_stds[i];
#ifdef DIAGNOSTIC
               if (le32toh(std->td.td_token) == TD_IS_FREE) {
                       printf("%s: freeing free TD %p\n", __func__, std);
                       return;
               }
               std->td.td_token = htole32(TD_IS_FREE);
#endif
               ux->ux_stds[i]->link.std = sc->sc_freetds;
               sc->sc_freetds = std;
       }
       mutex_exit(&sc->sc_lock);
}


Static void
uhci_reset_std_chain(uhci_softc_t *sc, struct usbd_xfer *xfer,
   int length, int isread, int *toggle, uhci_soft_td_t **lstd)
{
       struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
       struct usbd_pipe *pipe = xfer->ux_pipe;
       usb_dma_t *dma = &xfer->ux_dmabuf;
       uint16_t flags = xfer->ux_flags;
       uhci_soft_td_t *std, *prev;
       int len = length;
       int tog = *toggle;
       int maxp;
       uint32_t status;
       size_t i, offs;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(8, "xfer=%#jx len %jd isread %jd toggle %jd", (uintptr_t)xfer,
           len, isread, *toggle);

       KASSERT(len != 0 || (!isread && (flags & USBD_FORCE_SHORT_XFER)));

       maxp = UGETW(pipe->up_endpoint->ue_edesc->wMaxPacketSize);
       KASSERT(maxp != 0);

       int addr = xfer->ux_pipe->up_dev->ud_addr;
       int endpt = xfer->ux_pipe->up_endpoint->ue_edesc->bEndpointAddress;

       status = UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) | UHCI_TD_ACTIVE);
       if (pipe->up_dev->ud_speed == USB_SPEED_LOW)
               status |= UHCI_TD_LS;
       if (flags & USBD_SHORT_XFER_OK)
               status |= UHCI_TD_SPD;
       usb_syncmem(dma, 0, len,
           isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
       std = prev = NULL;
       for (offs = i = 0; len != 0 && i < uxfer->ux_nstd; i++, prev = std) {
               int l = len;
               std = uxfer->ux_stds[i];

               const bus_addr_t sbp = DMAADDR(dma, offs);
               const bus_addr_t ebp = DMAADDR(dma, offs + l - 1);
               if (((sbp ^ ebp) & ~PAGE_MASK) != 0)
                       l = PAGE_SIZE - (DMAADDR(dma, offs) & PAGE_MASK);

               if (l > maxp)
                       l = maxp;

               if (prev) {
                       prev->link.std = std;
                       prev->td.td_link = htole32(
                           std->physaddr | UHCI_PTR_VF | UHCI_PTR_TD
                           );
                       usb_syncmem(&prev->dma, prev->offs, sizeof(prev->td),
                           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               }

               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);

               std->td.td_link = htole32(UHCI_PTR_T | UHCI_PTR_VF | UHCI_PTR_TD);
               std->td.td_status = htole32(status);
               std->td.td_token = htole32(
                   UHCI_TD_SET_ENDPT(UE_GET_ADDR(endpt)) |
                   UHCI_TD_SET_DEVADDR(addr) |
                   UHCI_TD_SET_PID(isread ? UHCI_TD_PID_IN : UHCI_TD_PID_OUT) |
                   UHCI_TD_SET_DT(tog) |
                   UHCI_TD_SET_MAXLEN(l)
                   );
               std->td.td_buffer = htole32(DMAADDR(dma, offs));

               std->link.std = NULL;

               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               tog ^= 1;

               offs += l;
               len -= l;
       }
       KASSERTMSG(len == 0, "xfer %p alen %d len %d mps %d ux_nqtd %zu i %zu",
           xfer, length, len, maxp, uxfer->ux_nstd, i);

       if (!isread &&
           (flags & USBD_FORCE_SHORT_XFER) &&
           length % maxp == 0) {
               /* Force a 0 length transfer at the end. */
               KASSERTMSG(i < uxfer->ux_nstd, "i=%zu nstd=%zu", i,
                   uxfer->ux_nstd);
               std = uxfer->ux_stds[i++];

               std->td.td_link = htole32(UHCI_PTR_T | UHCI_PTR_VF | UHCI_PTR_TD);
               std->td.td_status = htole32(status);
               std->td.td_token = htole32(
                   UHCI_TD_SET_ENDPT(UE_GET_ADDR(endpt)) |
                   UHCI_TD_SET_DEVADDR(addr) |
                   UHCI_TD_SET_PID(UHCI_TD_PID_OUT) |
                   UHCI_TD_SET_DT(tog) |
                   UHCI_TD_SET_MAXLEN(0)
                   );
               std->td.td_buffer = 0;
               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

               std->link.std = NULL;
               if (prev) {
                       prev->link.std = std;
                       prev->td.td_link = htole32(
                           std->physaddr | UHCI_PTR_VF | UHCI_PTR_TD
                           );
                       usb_syncmem(&prev->dma, prev->offs, sizeof(prev->td),
                           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               }
               tog ^= 1;
       }
       *lstd = std;
       *toggle = tog;
}

void
uhci_device_clear_toggle(struct usbd_pipe *pipe)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       upipe->nexttoggle = 0;
}

void
uhci_noop(struct usbd_pipe *pipe)
{
}

int
uhci_device_bulk_init(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
       usb_endpoint_descriptor_t *ed = xfer->ux_pipe->up_endpoint->ue_edesc;
       int endpt = ed->bEndpointAddress;
       int isread = UE_GET_DIR(endpt) == UE_DIR_IN;
       int len = xfer->ux_bufsize;
       int err = 0;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(3, "xfer=%#jx len=%jd flags=%jd", (uintptr_t)xfer, len,
           xfer->ux_flags, 0);

       if (sc->sc_dying)
               return USBD_IOERROR;

       KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));

       uxfer->ux_type = UX_BULK;
       err = uhci_alloc_std_chain(sc, xfer, len, isread, &uxfer->ux_stdstart);
       if (err)
               return err;

#ifdef UHCI_DEBUG
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_tds(uxfer->ux_stdstart);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       return 0;
}

Static void
uhci_device_bulk_fini(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);

       KASSERT(ux->ux_type == UX_BULK);

       if (ux->ux_nstd) {
               uhci_free_stds(sc, ux);
               kmem_free(ux->ux_stds, sizeof(uhci_soft_td_t *) * ux->ux_nstd);
       }
}

usbd_status
uhci_device_bulk_transfer(struct usbd_xfer *xfer)
{

       /* Pipe isn't running, so start it first.  */
       return uhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
}

usbd_status
uhci_device_bulk_start(struct usbd_xfer *xfer)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       uhci_soft_td_t *data, *dataend;
       uhci_soft_qh_t *sqh;
       int len;
       int endpt;
       int isread;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(3, "xfer=%#jx len=%jd flags=%jd", (uintptr_t)xfer,
           xfer->ux_length, xfer->ux_flags, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       if (sc->sc_dying)
               return USBD_IOERROR;

       KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
       KASSERT(xfer->ux_length <= xfer->ux_bufsize);

       len = xfer->ux_length;
       endpt = upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
       isread = UE_GET_DIR(endpt) == UE_DIR_IN;
       sqh = upipe->bulk.sqh;

       uhci_reset_std_chain(sc, xfer, len, isread, &upipe->nexttoggle,
           &dataend);

       data = ux->ux_stdstart;
       ux->ux_stdend = dataend;
       dataend->td.td_status |= htole32(UHCI_TD_IOC);
       usb_syncmem(&dataend->dma,
           dataend->offs + offsetof(uhci_td_t, td_status),
           sizeof(dataend->td.td_status),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

#ifdef UHCI_DEBUG
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               DPRINTFN(10, "before transfer", 0, 0, 0, 0);
               uhci_dump_tds(data);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       KASSERT(ux->ux_isdone);
#ifdef DIAGNOSTIC
       ux->ux_isdone = false;
#endif

       sqh->elink = data;
       sqh->qh.qh_elink = htole32(data->physaddr | UHCI_PTR_TD);
       /* uhci_add_bulk() will do usb_syncmem(sqh) */

       uhci_add_bulk(sc, sqh);
       uhci_add_intr_list(sc, ux);
       xfer->ux_status = USBD_IN_PROGRESS;
       usbd_xfer_schedule_timeout(xfer);

       return USBD_IN_PROGRESS;
}

/* Abort a device bulk request. */
void
uhci_device_bulk_abort(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       usbd_xfer_abort(xfer);
}

/*
* To allow the hardware time to notice we simply wait.
*/
Static void
uhci_abortx(struct usbd_xfer *xfer)
{
       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       uhci_soft_td_t *std;

       DPRINTFN(1,"xfer=%#jx", (uintptr_t)xfer, 0, 0, 0);

       KASSERT(mutex_owned(&sc->sc_lock));
       ASSERT_SLEEPABLE();

       KASSERTMSG((xfer->ux_status == USBD_CANCELLED ||
               xfer->ux_status == USBD_TIMEOUT),
           "bad abort status: %d", xfer->ux_status);

       /*
        * If we're dying, skip the hardware action and just notify the
        * software that we're done.
        */
       if (sc->sc_dying) {
               DPRINTFN(4, "xfer %#jx dying %ju", (uintptr_t)xfer,
                   xfer->ux_status, 0, 0);
               goto dying;
       }

       /*
        * HC Step 1: Make interrupt routine and hardware ignore xfer.
        */
       uhci_del_intr_list(sc, ux);

       DPRINTF("stop ux=%#jx", (uintptr_t)ux, 0, 0, 0);
       for (std = ux->ux_stdstart; std != NULL; std = std->link.std) {
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               std->td.td_status &= htole32(~(UHCI_TD_ACTIVE | UHCI_TD_IOC));
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       }

       /*
        * HC Step 2: Wait until we know hardware has finished any possible
        * use of the xfer.
        */
       /* Hardware finishes in 1ms */
       usb_delay_ms_locked(upipe->pipe.up_dev->ud_bus, 2, &sc->sc_lock);
dying:
#ifdef DIAGNOSTIC
       ux->ux_isdone = true;
#endif
       DPRINTFN(14, "end", 0, 0, 0, 0);

       KASSERT(mutex_owned(&sc->sc_lock));
}

/* Close a device bulk pipe. */
void
uhci_device_bulk_close(struct usbd_pipe *pipe)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);

       KASSERT(mutex_owned(&sc->sc_lock));

       uhci_free_sqh(sc, upipe->bulk.sqh);

       pipe->up_endpoint->ue_toggle = upipe->nexttoggle;
}

int
uhci_device_ctrl_init(struct usbd_xfer *xfer)
{
       struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       usb_device_request_t *req = &xfer->ux_request;
       struct usbd_device *dev = upipe->pipe.up_dev;
       uhci_softc_t *sc = dev->ud_bus->ub_hcpriv;
       uhci_soft_td_t *data = NULL;
       int len;
       usbd_status err;
       int isread;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(3, "xfer=%#jx len=%jd, addr=%jd, endpt=%jd",
           (uintptr_t)xfer, xfer->ux_bufsize, dev->ud_addr,
           upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress);

       isread = req->bmRequestType & UT_READ;
       len = xfer->ux_bufsize;

       uxfer->ux_type = UX_CTRL;
       /* Set up data transaction */
       if (len != 0) {
               err = uhci_alloc_std_chain(sc, xfer, len, isread, &data);
               if (err)
                       return err;
       }
       /* Set up interrupt info. */
       uxfer->ux_setup = upipe->ctrl.setup;
       uxfer->ux_stat = upipe->ctrl.stat;
       uxfer->ux_data = data;

       return 0;
}

Static void
uhci_device_ctrl_fini(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);

       KASSERT(ux->ux_type == UX_CTRL);

       if (ux->ux_nstd) {
               uhci_free_stds(sc, ux);
               kmem_free(ux->ux_stds, sizeof(uhci_soft_td_t *) * ux->ux_nstd);
       }
}

usbd_status
uhci_device_ctrl_transfer(struct usbd_xfer *xfer)
{

       /* Pipe isn't running, so start it first.  */
       return uhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
}

usbd_status
uhci_device_ctrl_start(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *uxfer = UHCI_XFER2UXFER(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       usb_device_request_t *req = &xfer->ux_request;
       struct usbd_device *dev = upipe->pipe.up_dev;
       int addr = dev->ud_addr;
       int endpt = upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
       uhci_soft_td_t *setup, *stat, *next, *dataend;
       uhci_soft_qh_t *sqh;
       int len;
       int isread;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       if (sc->sc_dying)
               return USBD_IOERROR;

       KASSERT(xfer->ux_rqflags & URQ_REQUEST);

       DPRINTFN(3, "type=0x%02jx, request=0x%02jx, "
           "wValue=0x%04jx, wIndex=0x%04jx",
           req->bmRequestType, req->bRequest, UGETW(req->wValue),
           UGETW(req->wIndex));
       DPRINTFN(3, "len=%jd, addr=%jd, endpt=%jd",
           UGETW(req->wLength), dev->ud_addr, endpt, 0);

       isread = req->bmRequestType & UT_READ;
       len = UGETW(req->wLength);

       setup = upipe->ctrl.setup;
       stat = upipe->ctrl.stat;
       sqh = upipe->ctrl.sqh;

       memcpy(KERNADDR(&upipe->ctrl.reqdma, 0), req, sizeof(*req));
       usb_syncmem(&upipe->ctrl.reqdma, 0, sizeof(*req), BUS_DMASYNC_PREWRITE);

       /* Set up data transaction */
       if (len != 0) {
               upipe->nexttoggle = 1;
               next = uxfer->ux_data;
               uhci_reset_std_chain(sc, xfer, len, isread,
                   &upipe->nexttoggle, &dataend);
               dataend->link.std = stat;
               dataend->td.td_link = htole32(stat->physaddr | UHCI_PTR_TD);
               usb_syncmem(&dataend->dma,
                   dataend->offs + offsetof(uhci_td_t, td_link),
                   sizeof(dataend->td.td_link),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       } else {
               next = stat;
       }

       const uint32_t status = UHCI_TD_ZERO_ACTLEN(
           UHCI_TD_SET_ERRCNT(3) |
           UHCI_TD_ACTIVE |
           (dev->ud_speed == USB_SPEED_LOW ? UHCI_TD_LS : 0)
           );
       setup->link.std = next;
       setup->td.td_link = htole32(next->physaddr | UHCI_PTR_TD);
       setup->td.td_status = htole32(status);
       setup->td.td_token = htole32(UHCI_TD_SETUP(sizeof(*req), endpt, addr));
       setup->td.td_buffer = htole32(DMAADDR(&upipe->ctrl.reqdma, 0));

       usb_syncmem(&setup->dma, setup->offs, sizeof(setup->td),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       stat->link.std = NULL;
       stat->td.td_link = htole32(UHCI_PTR_T);
       stat->td.td_status = htole32(status | UHCI_TD_IOC);
       stat->td.td_token =
               htole32(isread ? UHCI_TD_OUT(0, endpt, addr, 1) :
                                UHCI_TD_IN (0, endpt, addr, 1));
       stat->td.td_buffer = htole32(0);
       usb_syncmem(&stat->dma, stat->offs, sizeof(stat->td),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

#ifdef UHCI_DEBUG
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               DPRINTF("before transfer", 0, 0, 0, 0);
               uhci_dump_tds(setup);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       /* Set up interrupt info. */
       uxfer->ux_setup = setup;
       uxfer->ux_stat = stat;
       KASSERT(uxfer->ux_isdone);
#ifdef DIAGNOSTIC
       uxfer->ux_isdone = false;
#endif

       sqh->elink = setup;
       sqh->qh.qh_elink = htole32(setup->physaddr | UHCI_PTR_TD);
       /* uhci_add_?s_ctrl() will do usb_syncmem(sqh) */

       if (dev->ud_speed == USB_SPEED_LOW)
               uhci_add_ls_ctrl(sc, sqh);
       else
               uhci_add_hs_ctrl(sc, sqh);
       uhci_add_intr_list(sc, uxfer);
#ifdef UHCI_DEBUG
       if (uhcidebug >= 12) {
               uhci_soft_td_t *std;
               uhci_soft_qh_t *xqh;
               uhci_soft_qh_t *sxqh;
               int maxqh = 0;
               uhci_physaddr_t link;
               DPRINTFN(12, "--- dump start ---", 0, 0, 0, 0);
               DPRINTFN(12, "follow from [0]", 0, 0, 0, 0);
               for (std = sc->sc_vframes[0].htd, link = 0;
                    (link & UHCI_PTR_QH) == 0;
                    std = std->link.std) {
                       link = le32toh(std->td.td_link);
                       uhci_dump_td(std);
               }
               sxqh = (uhci_soft_qh_t *)std;
               uhci_dump_qh(sxqh);
               for (xqh = sxqh;
                    xqh != NULL;
                    xqh = (maxqh++ == 5 || xqh->hlink == sxqh ||
                       xqh->hlink == xqh ? NULL : xqh->hlink)) {
                       uhci_dump_qh(xqh);
               }
               DPRINTFN(12, "Enqueued QH:", 0, 0, 0, 0);
               uhci_dump_qh(sqh);
               uhci_dump_tds(sqh->elink);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif
       xfer->ux_status = USBD_IN_PROGRESS;
       usbd_xfer_schedule_timeout(xfer);

       return USBD_IN_PROGRESS;
}

int
uhci_device_intr_init(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       usb_endpoint_descriptor_t *ed = xfer->ux_pipe->up_endpoint->ue_edesc;
       int endpt = ed->bEndpointAddress;
       int isread = UE_GET_DIR(endpt) == UE_DIR_IN;
       int len = xfer->ux_bufsize;
       int err;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       DPRINTFN(3, "xfer=%#jx len=%jd flags=%jd", (uintptr_t)xfer,
           xfer->ux_length, xfer->ux_flags, 0);

       KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
       KASSERT(len != 0);

       ux->ux_type = UX_INTR;
       ux->ux_nstd = 0;
       err = uhci_alloc_std_chain(sc, xfer, len, isread, &ux->ux_stdstart);

       return err;
}

Static void
uhci_device_intr_fini(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);

       KASSERT(ux->ux_type == UX_INTR);

       if (ux->ux_nstd) {
               uhci_free_stds(sc, ux);
               kmem_free(ux->ux_stds, sizeof(uhci_soft_td_t *) * ux->ux_nstd);
       }
}

usbd_status
uhci_device_intr_transfer(struct usbd_xfer *xfer)
{

       /* Pipe isn't running, so start it first.  */
       return uhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
}

usbd_status
uhci_device_intr_start(struct usbd_xfer *xfer)
{
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       uhci_soft_td_t *data, *dataend;
       uhci_soft_qh_t *sqh;
       int isread, endpt;
       int i;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       DPRINTFN(3, "xfer=%#jx len=%jd flags=%jd", (uintptr_t)xfer,
           xfer->ux_length, xfer->ux_flags, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       if (sc->sc_dying)
               return USBD_IOERROR;

       KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
       KASSERT(xfer->ux_length <= xfer->ux_bufsize);

       endpt = upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
       isread = UE_GET_DIR(endpt) == UE_DIR_IN;

       data = ux->ux_stdstart;

       KASSERT(ux->ux_isdone);
#ifdef DIAGNOSTIC
       ux->ux_isdone = false;
#endif

       /* Take lock to protect nexttoggle */
       uhci_reset_std_chain(sc, xfer, xfer->ux_length, isread,
           &upipe->nexttoggle, &dataend);

       dataend->td.td_status |= htole32(UHCI_TD_IOC);
       usb_syncmem(&dataend->dma,
           dataend->offs + offsetof(uhci_td_t, td_status),
           sizeof(dataend->td.td_status),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       ux->ux_stdend = dataend;

#ifdef UHCI_DEBUG
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_tds(data);
               uhci_dump_qh(upipe->intr.qhs[0]);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       DPRINTFN(10, "qhs[0]=%#jx", (uintptr_t)upipe->intr.qhs[0], 0, 0, 0);
       for (i = 0; i < upipe->intr.npoll; i++) {
               sqh = upipe->intr.qhs[i];
               sqh->elink = data;
               sqh->qh.qh_elink = htole32(data->physaddr | UHCI_PTR_TD);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       }
       uhci_add_intr_list(sc, ux);
       xfer->ux_status = USBD_IN_PROGRESS;

#ifdef UHCI_DEBUG
       if (uhcidebug >= 10) {
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_tds(data);
               uhci_dump_qh(upipe->intr.qhs[0]);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
       }
#endif

       return USBD_IN_PROGRESS;
}

/* Abort a device control request. */
void
uhci_device_ctrl_abort(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       usbd_xfer_abort(xfer);
}

/* Close a device control pipe. */
void
uhci_device_ctrl_close(struct usbd_pipe *pipe)
{
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);

       uhci_free_sqh(sc, upipe->ctrl.sqh);
       uhci_free_std_locked(sc, upipe->ctrl.setup);
       uhci_free_std_locked(sc, upipe->ctrl.stat);

       usb_freemem(&upipe->ctrl.reqdma);
}

/* Abort a device interrupt request. */
void
uhci_device_intr_abort(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTF("xfer=%#jx", (uintptr_t)xfer, 0, 0, 0);

       usbd_xfer_abort(xfer);
}

/* Close a device interrupt pipe. */
void
uhci_device_intr_close(struct usbd_pipe *pipe)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       int i, npoll;

       KASSERT(mutex_owned(&sc->sc_lock));

       /* Unlink descriptors from controller data structures. */
       npoll = upipe->intr.npoll;
       for (i = 0; i < npoll; i++)
               uhci_remove_intr(sc, upipe->intr.qhs[i]);

       /*
        * We now have to wait for any activity on the physical
        * descriptors to stop.
        */
       usb_delay_ms_locked(&sc->sc_bus, 2, &sc->sc_lock);

       for (i = 0; i < npoll; i++)
               uhci_free_sqh(sc, upipe->intr.qhs[i]);
       kmem_free(upipe->intr.qhs, npoll * sizeof(uhci_soft_qh_t *));
}

int
uhci_device_isoc_init(struct usbd_xfer *xfer)
{
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);

       KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
       KASSERT(xfer->ux_nframes != 0);
       KASSERT(ux->ux_isdone);

       ux->ux_type = UX_ISOC;
       return 0;
}

Static void
uhci_device_isoc_fini(struct usbd_xfer *xfer)
{
       struct uhci_xfer *ux __diagused = UHCI_XFER2UXFER(xfer);

       KASSERT(ux->ux_type == UX_ISOC);
}

usbd_status
uhci_device_isoc_transfer(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(5, "xfer=%#jx", (uintptr_t)xfer, 0, 0, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       /* insert into schedule, */

       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       struct isoc *isoc = &upipe->isoc;
       uhci_soft_td_t *std = NULL;
       uint32_t buf, len, status, offs;
       int i, next, nframes;
       int rd = UE_GET_DIR(upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress) == UE_DIR_IN;

       DPRINTFN(5, "used=%jd next=%jd xfer=%#jx nframes=%jd",
           isoc->inuse, isoc->next, (uintptr_t)xfer, xfer->ux_nframes);

       if (sc->sc_dying)
               return USBD_IOERROR;

       if (xfer->ux_status == USBD_IN_PROGRESS) {
               /* This request has already been entered into the frame list */
               printf("%s: xfer=%p in frame list\n", __func__, xfer);
               /* XXX */
       }

#ifdef DIAGNOSTIC
       if (isoc->inuse >= UHCI_VFRAMELIST_COUNT)
               printf("%s: overflow!\n", __func__);
#endif

       KASSERT(xfer->ux_nframes != 0);

       if (xfer->ux_length)
               usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
                   rd ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);

       next = isoc->next;
       if (next == -1) {
               /* Not in use yet, schedule it a few frames ahead. */
               next = (UREAD2(sc, UHCI_FRNUM) + 3) % UHCI_VFRAMELIST_COUNT;
               DPRINTFN(2, "start next=%jd", next, 0, 0, 0);
       }

       xfer->ux_status = USBD_IN_PROGRESS;
       ux->ux_curframe = next;

       offs = 0;
       status = UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(0) |
                                    UHCI_TD_ACTIVE |
                                    UHCI_TD_IOS);
       nframes = xfer->ux_nframes;
       for (i = 0; i < nframes; i++) {
               buf = DMAADDR(&xfer->ux_dmabuf, offs);
               std = isoc->stds[next];
               if (++next >= UHCI_VFRAMELIST_COUNT)
                       next = 0;
               len = xfer->ux_frlengths[i];

               KASSERTMSG(len <= __SHIFTOUT_MASK(UHCI_TD_MAXLEN_MASK),
                   "len %d", len);
               std->td.td_buffer = htole32(buf);
               usb_syncmem(&xfer->ux_dmabuf, offs, len,
                   rd ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
               if (i == nframes - 1)
                       status |= UHCI_TD_IOC;
               std->td.td_status = htole32(status);
               std->td.td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
               std->td.td_token |= htole32(UHCI_TD_SET_MAXLEN(len));
               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
#ifdef UHCI_DEBUG
               if (uhcidebug >= 5) {
                       DPRINTF("--- dump start ---", 0, 0, 0, 0);
                       DPRINTF("TD %jd", i, 0, 0, 0);
                       uhci_dump_td(std);
                       DPRINTF("--- dump end ---", 0, 0, 0, 0);
               }
#endif
               offs += len;
               const bus_addr_t bend __diagused =
                   DMAADDR(&xfer->ux_dmabuf, offs - 1);

               KASSERT(((buf ^ bend) & ~PAGE_MASK) == 0);
       }
       isoc->next = next;
       isoc->inuse += xfer->ux_nframes;

       /* Set up interrupt info. */
       ux->ux_stdstart = std;
       ux->ux_stdend = std;

       KASSERT(ux->ux_isdone);
#ifdef DIAGNOSTIC
       ux->ux_isdone = false;
#endif
       uhci_add_intr_list(sc, ux);

       return USBD_IN_PROGRESS;
}

void
uhci_device_isoc_abort(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       uhci_soft_td_t **stds = upipe->isoc.stds;
       uhci_soft_td_t *std;
       int i, n, nframes, maxlen, len;

       KASSERT(mutex_owned(&sc->sc_lock));

       /* Transfer is already done. */
       if (xfer->ux_status != USBD_NOT_STARTED &&
           xfer->ux_status != USBD_IN_PROGRESS) {
               return;
       }

       /* Give xfer the requested abort code. */
       xfer->ux_status = USBD_CANCELLED;

       /* make hardware ignore it, */
       nframes = xfer->ux_nframes;
       n = ux->ux_curframe;
       maxlen = 0;
       for (i = 0; i < nframes; i++) {
               std = stds[n];
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               std->td.td_status &= htole32(~(UHCI_TD_ACTIVE | UHCI_TD_IOC));
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_token),
                   sizeof(std->td.td_token),
                   BUS_DMASYNC_POSTWRITE);
               len = UHCI_TD_GET_MAXLEN(le32toh(std->td.td_token));
               if (len > maxlen)
                       maxlen = len;
               if (++n >= UHCI_VFRAMELIST_COUNT)
                       n = 0;
       }

       /* and wait until we are sure the hardware has finished. */
       delay(maxlen);

#ifdef DIAGNOSTIC
       ux->ux_isdone = true;
#endif
       /* Remove from interrupt list. */
       uhci_del_intr_list(sc, ux);

       /* Run callback. */
       usb_transfer_complete(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));
}

void
uhci_device_isoc_close(struct usbd_pipe *pipe)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       uhci_soft_td_t *std, *vstd;
       struct isoc *isoc;
       int i;

       KASSERT(mutex_owned(&sc->sc_lock));

       /*
        * Make sure all TDs are marked as inactive.
        * Wait for completion.
        * Unschedule.
        * Deallocate.
        */
       isoc = &upipe->isoc;

       for (i = 0; i < UHCI_VFRAMELIST_COUNT; i++) {
               std = isoc->stds[i];
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
               std->td.td_status &= htole32(~UHCI_TD_ACTIVE);
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_status),
                   sizeof(std->td.td_status),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       }
       /* wait for completion */
       usb_delay_ms_locked(&sc->sc_bus, 2, &sc->sc_lock);

       for (i = 0; i < UHCI_VFRAMELIST_COUNT; i++) {
               std = isoc->stds[i];
               for (vstd = sc->sc_vframes[i].htd;
                    vstd != NULL && vstd->link.std != std;
                    vstd = vstd->link.std)
                       ;
               if (vstd == NULL) {
                       /*panic*/
                       printf("%s: %p not found\n", __func__, std);
                       mutex_exit(&sc->sc_lock);
                       return;
               }
               vstd->link = std->link;
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_link),
                   sizeof(std->td.td_link),
                   BUS_DMASYNC_POSTWRITE);
               vstd->td.td_link = std->td.td_link;
               usb_syncmem(&vstd->dma,
                   vstd->offs + offsetof(uhci_td_t, td_link),
                   sizeof(vstd->td.td_link),
                   BUS_DMASYNC_PREWRITE);
               uhci_free_std_locked(sc, std);
       }

       kmem_free(isoc->stds, UHCI_VFRAMELIST_COUNT * sizeof(uhci_soft_td_t *));
}

usbd_status
uhci_setup_isoc(struct usbd_pipe *pipe)
{
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       int addr = upipe->pipe.up_dev->ud_addr;
       int endpt = upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
       int rd = UE_GET_DIR(endpt) == UE_DIR_IN;
       uhci_soft_td_t *std, *vstd;
       uint32_t token;
       struct isoc *isoc;
       int i;

       isoc = &upipe->isoc;

       isoc->stds = kmem_alloc(
           UHCI_VFRAMELIST_COUNT * sizeof(uhci_soft_td_t *), KM_SLEEP);
       if (isoc->stds == NULL)
               return USBD_NOMEM;

       token = rd ? UHCI_TD_IN (0, endpt, addr, 0) :
                    UHCI_TD_OUT(0, endpt, addr, 0);

       /* Allocate the TDs and mark as inactive; */
       for (i = 0; i < UHCI_VFRAMELIST_COUNT; i++) {
               std = uhci_alloc_std(sc);
               if (std == 0)
                       goto bad;
               std->td.td_status = htole32(UHCI_TD_IOS); /* iso, inactive */
               std->td.td_token = htole32(token);
               usb_syncmem(&std->dma, std->offs, sizeof(std->td),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               isoc->stds[i] = std;
       }

       mutex_enter(&sc->sc_lock);

       /* Insert TDs into schedule. */
       for (i = 0; i < UHCI_VFRAMELIST_COUNT; i++) {
               std = isoc->stds[i];
               vstd = sc->sc_vframes[i].htd;
               usb_syncmem(&vstd->dma,
                   vstd->offs + offsetof(uhci_td_t, td_link),
                   sizeof(vstd->td.td_link),
                   BUS_DMASYNC_POSTWRITE);
               std->link = vstd->link;
               std->td.td_link = vstd->td.td_link;
               usb_syncmem(&std->dma,
                   std->offs + offsetof(uhci_td_t, td_link),
                   sizeof(std->td.td_link),
                   BUS_DMASYNC_PREWRITE);
               vstd->link.std = std;
               vstd->td.td_link = htole32(std->physaddr | UHCI_PTR_TD);
               usb_syncmem(&vstd->dma,
                   vstd->offs + offsetof(uhci_td_t, td_link),
                   sizeof(vstd->td.td_link),
                   BUS_DMASYNC_PREWRITE);
       }
       mutex_exit(&sc->sc_lock);

       isoc->next = -1;
       isoc->inuse = 0;

       return USBD_NORMAL_COMPLETION;

bad:
       while (--i >= 0)
               uhci_free_std(sc, isoc->stds[i]);
       kmem_free(isoc->stds, UHCI_VFRAMELIST_COUNT * sizeof(uhci_soft_td_t *));
       return USBD_NOMEM;
}

void
uhci_device_isoc_done(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       struct uhci_xfer *ux = UHCI_XFER2UXFER(xfer);
       int rd = UE_GET_DIR(upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress) == UE_DIR_IN;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(4, "length=%jd, ux_state=0x%08jx",
           xfer->ux_actlen, xfer->ux_state, 0, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

#ifdef DIAGNOSTIC
       if (ux->ux_stdend == NULL) {
               printf("%s: xfer=%p stdend==NULL\n", __func__, xfer);
#ifdef UHCI_DEBUG
               DPRINTF("--- dump start ---", 0, 0, 0, 0);
               uhci_dump_ii(ux);
               DPRINTF("--- dump end ---", 0, 0, 0, 0);
#endif
               return;
       }
#endif

       /* Turn off the interrupt since it is active even if the TD is not. */
       usb_syncmem(&ux->ux_stdend->dma,
           ux->ux_stdend->offs + offsetof(uhci_td_t, td_status),
           sizeof(ux->ux_stdend->td.td_status),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       ux->ux_stdend->td.td_status &= htole32(~UHCI_TD_IOC);
       usb_syncmem(&ux->ux_stdend->dma,
           ux->ux_stdend->offs + offsetof(uhci_td_t, td_status),
           sizeof(ux->ux_stdend->td.td_status),
           BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

       if (xfer->ux_length)
               usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
                   rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
}

void
uhci_device_intr_done(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc __diagused = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       uhci_soft_qh_t *sqh;
       int i, npoll;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(5, "length=%jd", xfer->ux_actlen, 0, 0, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       npoll = upipe->intr.npoll;
       for (i = 0; i < npoll; i++) {
               sqh = upipe->intr.qhs[i];
               sqh->elink = NULL;
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
       }
       const int endpt = upipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
       const bool isread = UE_GET_DIR(endpt) == UE_DIR_IN;
       usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
           isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
}

/* Deallocate request data structures */
void
uhci_device_ctrl_done(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       int len = UGETW(xfer->ux_request.wLength);
       int isread = (xfer->ux_request.bmRequestType & UT_READ);

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
       KASSERT(xfer->ux_rqflags & URQ_REQUEST);

       /* XXXNH move to uhci_idone??? */
       if (upipe->pipe.up_dev->ud_speed == USB_SPEED_LOW)
               uhci_remove_ls_ctrl(sc, upipe->ctrl.sqh);
       else
               uhci_remove_hs_ctrl(sc, upipe->ctrl.sqh);

       if (len) {
               usb_syncmem(&xfer->ux_dmabuf, 0, len,
                   isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
       }
       usb_syncmem(&upipe->ctrl.reqdma, 0,
           sizeof(usb_device_request_t),  BUS_DMASYNC_POSTWRITE);

       DPRINTF("length=%jd", xfer->ux_actlen, 0, 0, 0);
}

/* Deallocate request data structures */
void
uhci_device_bulk_done(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(xfer->ux_pipe);
       usb_endpoint_descriptor_t *ed = xfer->ux_pipe->up_endpoint->ue_edesc;
       int endpt = ed->bEndpointAddress;
       int isread = UE_GET_DIR(endpt) == UE_DIR_IN;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(5, "xfer=%#jx sc=%#jx upipe=%#jx", (uintptr_t)xfer,
           (uintptr_t)sc, (uintptr_t)upipe, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       uhci_remove_bulk(sc, upipe->bulk.sqh);

       if (xfer->ux_length) {
               usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
                   isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
       }

       DPRINTFN(5, "length=%jd", xfer->ux_actlen, 0, 0, 0);
}

/* Add interrupt QH, called with vflock. */
void
uhci_add_intr(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       struct uhci_vframe *vf = &sc->sc_vframes[sqh->pos];
       uhci_soft_qh_t *eqh;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(4, "n=%jd sqh=%#jx", sqh->pos, (uintptr_t)sqh, 0, 0);

       eqh = vf->eqh;
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_POSTWRITE);
       sqh->hlink       = eqh->hlink;
       sqh->qh.qh_hlink = eqh->qh.qh_hlink;
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(sqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       eqh->hlink       = sqh;
       eqh->qh.qh_hlink = htole32(sqh->physaddr | UHCI_PTR_QH);
       usb_syncmem(&eqh->dma, eqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(eqh->qh.qh_hlink), BUS_DMASYNC_PREWRITE);
       vf->eqh = sqh;
       vf->bandwidth++;
}

/* Remove interrupt QH. */
void
uhci_remove_intr(uhci_softc_t *sc, uhci_soft_qh_t *sqh)
{
       struct uhci_vframe *vf = &sc->sc_vframes[sqh->pos];
       uhci_soft_qh_t *pqh;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(4, "n=%jd sqh=%#jx", sqh->pos, (uintptr_t)sqh, 0, 0);

       /* See comment in uhci_remove_ctrl() */

       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_elink),
           sizeof(sqh->qh.qh_elink),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       if (!(sqh->qh.qh_elink & htole32(UHCI_PTR_T))) {
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               delay(UHCI_QH_REMOVE_DELAY);
       }

       pqh = uhci_find_prev_qh(vf->hqh, sqh);
       usb_syncmem(&sqh->dma, sqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(sqh->qh.qh_hlink),
           BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
       pqh->hlink       = sqh->hlink;
       pqh->qh.qh_hlink = sqh->qh.qh_hlink;
       usb_syncmem(&pqh->dma, pqh->offs + offsetof(uhci_qh_t, qh_hlink),
           sizeof(pqh->qh.qh_hlink),
           BUS_DMASYNC_PREWRITE);
       delay(UHCI_QH_REMOVE_DELAY);
       if (vf->eqh == sqh)
               vf->eqh = pqh;
       vf->bandwidth--;
}

usbd_status
uhci_device_setintr(uhci_softc_t *sc, struct uhci_pipe *upipe, int ival)
{
       uhci_soft_qh_t *sqh;
       int i, npoll;
       u_int bestbw, bw, bestoffs, offs;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTFN(2, "pipe=%#jx", (uintptr_t)upipe, 0, 0, 0);
       if (ival == 0) {
               printf("%s: 0 interval\n", __func__);
               return USBD_INVAL;
       }

       if (ival > UHCI_VFRAMELIST_COUNT)
               ival = UHCI_VFRAMELIST_COUNT;
       npoll = howmany(UHCI_VFRAMELIST_COUNT, ival);
       DPRINTF("ival=%jd npoll=%jd", ival, npoll, 0, 0);

       upipe->intr.npoll = npoll;
       upipe->intr.qhs =
               kmem_alloc(npoll * sizeof(uhci_soft_qh_t *), KM_SLEEP);

       /*
        * Figure out which offset in the schedule that has most
        * bandwidth left over.
        */
#define MOD(i) ((i) & (UHCI_VFRAMELIST_COUNT-1))
       for (bestoffs = offs = 0, bestbw = ~0; offs < ival; offs++) {
               for (bw = i = 0; i < npoll; i++)
                       bw += sc->sc_vframes[MOD(i * ival + offs)].bandwidth;
               if (bw < bestbw) {
                       bestbw = bw;
                       bestoffs = offs;
               }
       }
       DPRINTF("bw=%jd offs=%jd", bestbw, bestoffs, 0, 0);
       for (i = 0; i < npoll; i++) {
               upipe->intr.qhs[i] = sqh = uhci_alloc_sqh(sc);
               sqh->elink = NULL;
               sqh->qh.qh_elink = htole32(UHCI_PTR_T);
               usb_syncmem(&sqh->dma,
                   sqh->offs + offsetof(uhci_qh_t, qh_elink),
                   sizeof(sqh->qh.qh_elink),
                   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
               sqh->pos = MOD(i * ival + bestoffs);
       }
#undef MOD

       mutex_enter(&sc->sc_lock);
       /* Enter QHs into the controller data structures. */
       for (i = 0; i < npoll; i++)
               uhci_add_intr(sc, upipe->intr.qhs[i]);
       mutex_exit(&sc->sc_lock);

       DPRINTFN(5, "returns %#jx", (uintptr_t)upipe, 0, 0, 0);

       return USBD_NORMAL_COMPLETION;
}

/* Open a new pipe. */
usbd_status
uhci_open(struct usbd_pipe *pipe)
{
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       struct usbd_bus *bus = pipe->up_dev->ud_bus;
       struct uhci_pipe *upipe = UHCI_PIPE2UPIPE(pipe);
       usb_endpoint_descriptor_t *ed = pipe->up_endpoint->ue_edesc;
       int ival;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTF("pipe=%#jx, addr=%jd, endpt=%jd (%jd)",
           (uintptr_t)pipe, pipe->up_dev->ud_addr, ed->bEndpointAddress,
           bus->ub_rhaddr);

       if (sc->sc_dying)
               return USBD_IOERROR;

       upipe->aborting = 0;
       /* toggle state needed for bulk endpoints */
       upipe->nexttoggle = pipe->up_endpoint->ue_toggle;

       if (pipe->up_dev->ud_addr == bus->ub_rhaddr) {
               switch (ed->bEndpointAddress) {
               case USB_CONTROL_ENDPOINT:
                       pipe->up_methods = &roothub_ctrl_methods;
                       break;
               case UE_DIR_IN | USBROOTHUB_INTR_ENDPT:
                       pipe->up_methods = &uhci_root_intr_methods;
                       break;
               default:
                       return USBD_INVAL;
               }
       } else {
               switch (ed->bmAttributes & UE_XFERTYPE) {
               case UE_CONTROL:
                       pipe->up_methods = &uhci_device_ctrl_methods;
                       upipe->ctrl.sqh = uhci_alloc_sqh(sc);
                       if (upipe->ctrl.sqh == NULL)
                               goto bad;
                       upipe->ctrl.setup = uhci_alloc_std(sc);
                       if (upipe->ctrl.setup == NULL) {
                               uhci_free_sqh(sc, upipe->ctrl.sqh);
                               goto bad;
                       }
                       upipe->ctrl.stat = uhci_alloc_std(sc);
                       if (upipe->ctrl.stat == NULL) {
                               uhci_free_sqh(sc, upipe->ctrl.sqh);
                               uhci_free_std(sc, upipe->ctrl.setup);
                               goto bad;
                       }
                       int err = usb_allocmem(sc->sc_bus.ub_dmatag,
                           sizeof(usb_device_request_t), 0,
                           USBMALLOC_COHERENT, &upipe->ctrl.reqdma);
                       if (err) {
                               uhci_free_sqh(sc, upipe->ctrl.sqh);
                               uhci_free_std(sc, upipe->ctrl.setup);
                               uhci_free_std(sc, upipe->ctrl.stat);
                               goto bad;
                       }
                       break;
               case UE_INTERRUPT:
                       pipe->up_methods = &uhci_device_intr_methods;
                       ival = pipe->up_interval;
                       if (ival == USBD_DEFAULT_INTERVAL)
                               ival = ed->bInterval;
                       return uhci_device_setintr(sc, upipe, ival);
               case UE_ISOCHRONOUS:
                       pipe->up_serialise = false;
                       pipe->up_methods = &uhci_device_isoc_methods;
                       return uhci_setup_isoc(pipe);
               case UE_BULK:
                       pipe->up_methods = &uhci_device_bulk_methods;
                       upipe->bulk.sqh = uhci_alloc_sqh(sc);
                       if (upipe->bulk.sqh == NULL)
                               goto bad;
                       break;
               }
       }
       return USBD_NORMAL_COMPLETION;

bad:
       return USBD_NOMEM;
}

/*
* Data structures and routines to emulate the root hub.
*/
/*
* The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
* enables the port, and also states that SET_FEATURE(PORT_ENABLE)
* should not be used by the USB subsystem.  As we cannot issue a
* SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
* will be enabled as part of the reset.
*
* On the VT83C572, the port cannot be successfully enabled until the
* outstanding "port enable change" and "connection status change"
* events have been reset.
*/
Static usbd_status
uhci_portreset(uhci_softc_t *sc, int index)
{
       int lim, port, x;
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       if (index == 1)
               port = UHCI_PORTSC1;
       else if (index == 2)
               port = UHCI_PORTSC2;
       else
               return USBD_IOERROR;

       x = URWMASK(UREAD2(sc, port));
       UWRITE2(sc, port, x | UHCI_PORTSC_PR);

       usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);

       DPRINTF("uhci port %jd reset, status0 = 0x%04jx", index,
           UREAD2(sc, port), 0, 0);

       x = URWMASK(UREAD2(sc, port));
       UWRITE2(sc, port, x & ~(UHCI_PORTSC_PR | UHCI_PORTSC_SUSP));

       delay(100);

       DPRINTF("uhci port %jd reset, status1 = 0x%04jx", index,
           UREAD2(sc, port), 0, 0);

       x = URWMASK(UREAD2(sc, port));
       UWRITE2(sc, port, x  | UHCI_PORTSC_PE);

       for (lim = 10; --lim > 0;) {
               usb_delay_ms(&sc->sc_bus, USB_PORT_RESET_DELAY);

               x = UREAD2(sc, port);
               DPRINTF("uhci port %jd iteration %ju, status = 0x%04jx", index,
                   lim, x, 0);

               if (!(x & UHCI_PORTSC_CCS)) {
                       /*
                        * No device is connected (or was disconnected
                        * during reset).  Consider the port reset.
                        * The delay must be long enough to ensure on
                        * the initial iteration that the device
                        * connection will have been registered.  50ms
                        * appears to be sufficient, but 20ms is not.
                        */
                       DPRINTFN(3, "uhci port %jd loop %ju, device detached",
                           index, lim, 0, 0);
                       break;
               }

               if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
                       /*
                        * Port enabled changed and/or connection
                        * status changed were set.  Reset either or
                        * both raised flags (by writing a 1 to that
                        * bit), and wait again for state to settle.
                        */
                       UWRITE2(sc, port, URWMASK(x) |
                               (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
                       continue;
               }

               if (x & UHCI_PORTSC_PE)
                       /* Port is enabled */
                       break;

               UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
       }

       DPRINTFN(3, "uhci port %jd reset, status2 = 0x%04jx", index,
           UREAD2(sc, port), 0, 0);

       if (lim <= 0) {
               DPRINTF("uhci port %jd reset timed out", index,
                   0, 0, 0);
               return USBD_TIMEOUT;
       }

       sc->sc_isreset = 1;
       return USBD_NORMAL_COMPLETION;
}

Static int
uhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req,
   void *buf, int buflen)
{
       uhci_softc_t *sc = UHCI_BUS2SC(bus);
       int port, x;
       int status, change, totlen = 0;
       uint16_t len, value, index;
       usb_port_status_t ps;
       usbd_status err;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       if (sc->sc_dying)
               return -1;

       DPRINTF("type=0x%02jx request=%02jx", req->bmRequestType,
           req->bRequest, 0, 0);

       len = UGETW(req->wLength);
       value = UGETW(req->wValue);
       index = UGETW(req->wIndex);

#define C(x,y) ((x) | ((y) << 8))
       switch (C(req->bRequest, req->bmRequestType)) {
       case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
               DPRINTF("wValue=0x%04jx", value, 0, 0, 0);
               if (len == 0)
                       break;
               switch (value) {
#define sd ((usb_string_descriptor_t *)buf)
               case C(2, UDESC_STRING):
                       /* Product */
                       totlen = usb_makestrdesc(sd, len, "UHCI root hub");
                       break;
#undef sd
               default:
                       /* default from usbroothub */
                       return buflen;
               }
               break;

       /* Hub requests */
       case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
               break;
       case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
               DPRINTF("UR_CLEAR_PORT_FEATURE port=%jd feature=%jd", index,
                   value, 0, 0);
               if (index == 1)
                       port = UHCI_PORTSC1;
               else if (index == 2)
                       port = UHCI_PORTSC2;
               else {
                       return -1;
               }
               switch(value) {
               case UHF_PORT_ENABLE:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
                       break;
               case UHF_PORT_SUSPEND:
                       x = URWMASK(UREAD2(sc, port));
                       if (!(x & UHCI_PORTSC_SUSP)) /* not suspended */
                               break;
                       UWRITE2(sc, port, x | UHCI_PORTSC_RD);
                       /* see USB2 spec ch. 7.1.7.7 */
                       usb_delay_ms(&sc->sc_bus, 20);
                       UWRITE2(sc, port, x & ~UHCI_PORTSC_SUSP);
                       /* 10ms resume delay must be provided by caller */
                       break;
               case UHF_PORT_RESET:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
                       break;
               case UHF_C_PORT_CONNECTION:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
                       break;
               case UHF_C_PORT_ENABLE:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
                       break;
               case UHF_C_PORT_OVER_CURRENT:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
                       break;
               case UHF_C_PORT_RESET:
                       sc->sc_isreset = 0;
                       break;
               case UHF_PORT_CONNECTION:
               case UHF_PORT_OVER_CURRENT:
               case UHF_PORT_POWER:
               case UHF_PORT_LOW_SPEED:
               case UHF_C_PORT_SUSPEND:
               default:
                       return -1;
               }
               break;
       case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
               if (index == 1)
                       port = UHCI_PORTSC1;
               else if (index == 2)
                       port = UHCI_PORTSC2;
               else {
                       return -1;
               }
               if (len > 0) {
                       *(uint8_t *)buf =
                           UHCI_PORTSC_GET_LS(UREAD2(sc, port));
                       totlen = 1;
               }
               break;
       case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
               if (len == 0)
                       break;
               if ((value & 0xff) != 0) {
                       return -1;
               }
               usb_hub_descriptor_t hubd;

               totlen = uimin(buflen, sizeof(hubd));
               memcpy(&hubd, buf, totlen);
               hubd.bNbrPorts = 2;
               memcpy(buf, &hubd, totlen);
               break;
       case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
               if (len != 4) {
                       return -1;
               }
               memset(buf, 0, len);
               totlen = len;
               break;
       case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
               if (index == 1)
                       port = UHCI_PORTSC1;
               else if (index == 2)
                       port = UHCI_PORTSC2;
               else {
                       return -1;
               }
               if (len != 4) {
                       return -1;
               }
               x = UREAD2(sc, port);
               status = change = 0;
               if (x & UHCI_PORTSC_CCS)
                       status |= UPS_CURRENT_CONNECT_STATUS;
               if (x & UHCI_PORTSC_CSC)
                       change |= UPS_C_CONNECT_STATUS;
               if (x & UHCI_PORTSC_PE)
                       status |= UPS_PORT_ENABLED;
               if (x & UHCI_PORTSC_POEDC)
                       change |= UPS_C_PORT_ENABLED;
               if (x & UHCI_PORTSC_OCI)
                       status |= UPS_OVERCURRENT_INDICATOR;
               if (x & UHCI_PORTSC_OCIC)
                       change |= UPS_C_OVERCURRENT_INDICATOR;
               if (x & UHCI_PORTSC_SUSP)
                       status |= UPS_SUSPEND;
               if (x & UHCI_PORTSC_LSDA)
                       status |= UPS_LOW_SPEED;
               status |= UPS_PORT_POWER;
               if (sc->sc_isreset)
                       change |= UPS_C_PORT_RESET;
               USETW(ps.wPortStatus, status);
               USETW(ps.wPortChange, change);
               totlen = uimin(len, sizeof(ps));
               memcpy(buf, &ps, totlen);
               break;
       case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
               return -1;
       case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
               break;
       case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
               if (index == 1)
                       port = UHCI_PORTSC1;
               else if (index == 2)
                       port = UHCI_PORTSC2;
               else {
                       return -1;
               }
               switch(value) {
               case UHF_PORT_ENABLE:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x | UHCI_PORTSC_PE);
                       break;
               case UHF_PORT_SUSPEND:
                       x = URWMASK(UREAD2(sc, port));
                       UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
                       break;
               case UHF_PORT_RESET:
                       err = uhci_portreset(sc, index);
                       if (err != USBD_NORMAL_COMPLETION)
                               return -1;
                       return 0;
               case UHF_PORT_POWER:
                       /* Pretend we turned on power */
                       return 0;
               case UHF_C_PORT_CONNECTION:
               case UHF_C_PORT_ENABLE:
               case UHF_C_PORT_OVER_CURRENT:
               case UHF_PORT_CONNECTION:
               case UHF_PORT_OVER_CURRENT:
               case UHF_PORT_LOW_SPEED:
               case UHF_C_PORT_SUSPEND:
               case UHF_C_PORT_RESET:
               default:
                       return -1;
               }
               break;
       default:
               /* default from usbroothub */
               DPRINTF("returning %jd (usbroothub default)",
                   buflen, 0, 0, 0);
               return buflen;
       }

       DPRINTF("returning %jd", totlen, 0, 0, 0);

       return totlen;
}

/* Abort a root interrupt request. */
void
uhci_root_intr_abort(struct usbd_xfer *xfer)
{
       uhci_softc_t *sc = UHCI_XFER2SC(xfer);

       KASSERT(mutex_owned(&sc->sc_lock));
       KASSERT(xfer->ux_pipe->up_intrxfer == xfer);

       /*
        * Try to stop the callout before it starts.  If we got in too
        * late, too bad; but if the callout had yet to run and time
        * out the xfer, cancel it ourselves.
        */
       callout_stop(&sc->sc_poll_handle);
       if (sc->sc_intr_xfer == NULL)
               return;

       KASSERT(sc->sc_intr_xfer == xfer);
       KASSERT(xfer->ux_status == USBD_IN_PROGRESS);
       xfer->ux_status = USBD_CANCELLED;
#ifdef DIAGNOSTIC
       UHCI_XFER2UXFER(xfer)->ux_isdone = true;
#endif
       usb_transfer_complete(xfer);
}

usbd_status
uhci_root_intr_transfer(struct usbd_xfer *xfer)
{

       /* Pipe isn't running, start first */
       return uhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
}

/* Start a transfer on the root interrupt pipe */
usbd_status
uhci_root_intr_start(struct usbd_xfer *xfer)
{
       struct usbd_pipe *pipe = xfer->ux_pipe;
       uhci_softc_t *sc = UHCI_PIPE2SC(pipe);
       unsigned int ival;

       UHCIHIST_FUNC(); UHCIHIST_CALLED();
       DPRINTF("xfer=%#jx len=%jd flags=%jd", (uintptr_t)xfer, xfer->ux_length,
           xfer->ux_flags, 0);

       KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

       if (sc->sc_dying)
               return USBD_IOERROR;

       KASSERT(sc->sc_intr_xfer == NULL);

       /* XXX temporary variable needed to avoid gcc3 warning */
       ival = xfer->ux_pipe->up_endpoint->ue_edesc->bInterval;
       sc->sc_ival = mstohz(ival);
       if (sc->sc_suspend == PWR_RESUME)
               callout_schedule(&sc->sc_poll_handle, sc->sc_ival);
       sc->sc_intr_xfer = xfer;
       xfer->ux_status = USBD_IN_PROGRESS;

       return USBD_IN_PROGRESS;
}

/* Close the root interrupt pipe. */
void
uhci_root_intr_close(struct usbd_pipe *pipe)
{
       uhci_softc_t *sc __diagused = UHCI_PIPE2SC(pipe);
       UHCIHIST_FUNC(); UHCIHIST_CALLED();

       KASSERT(mutex_owned(&sc->sc_lock));

       /*
        * The caller must arrange to have aborted the pipe already, so
        * there can be no intr xfer in progress.  The callout may
        * still be pending from a prior intr xfer -- if it has already
        * fired, it will see there is nothing to do, and do nothing.
        */
       KASSERT(sc->sc_intr_xfer == NULL);
       KASSERT(!callout_pending(&sc->sc_poll_handle));
}

You are viewing proxied material from ftp.icm.edu.pl. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.