/*
* Copyright (c) 2020 The NetBSD Foundation, Inc.
* Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg.
* Copyright (c) 2010 Minoura Makoto.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
#include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
#define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */
/*
* The maximum descriptor size is 2^15. Use that value as the end of
* descriptor chain terminator since it will never be a valid index
* in the descriptor table.
*/
#define VRING_DESC_CHAIN_END 32768
void
virtio_set_status(struct virtio_softc *sc, int status)
{
sc->sc_ops->set_status(sc, status);
}
/*
* Reset the device.
*/
/*
* To reset the device to a known state, do following:
* virtio_reset(sc); // this will stop the device activity
* <dequeue finished requests>; // virtio_dequeue() still can be called
* <revoke pending requests in the vqs if any>;
* virtio_reinit_start(sc); // dequeue prohibited
* newfeatures = virtio_negotiate_features(sc, requestedfeatures);
* <some other initialization>;
* virtio_reinit_end(sc); // device activated; enqueue allowed
* Once attached, feature negotiation can only be allowed after virtio_reset.
*/
void
virtio_reset(struct virtio_softc *sc)
{
virtio_device_reset(sc);
}
int
virtio_reinit_start(struct virtio_softc *sc)
{
int i, r;
virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
for (i = 0; i < sc->sc_nvqs; i++) {
int n;
struct virtqueue *vq = &sc->sc_vqs[i];
n = sc->sc_ops->read_queue_size(sc, vq->vq_index);
if (n == 0) /* vq disappeared */
continue;
if (n != vq->vq_num) {
panic("%s: virtqueue size changed, vq index %d\n",
device_xname(sc->sc_dev),
vq->vq_index);
}
virtio_reset_vq(sc, vq);
sc->sc_ops->setup_queue(sc, vq->vq_index,
vq->vq_dmamap->dm_segs[0].ds_addr);
}
r = sc->sc_ops->setup_interrupts(sc, 1);
if (r != 0)
goto fail;
/*
* The Virtio spec explicitly tells that reading and writing 8 bytes are not
* considered atomic and no triggers may be connected to reading or writing
* it. We access it using two 32 reads. See virtio spec 4.1.3.1.
*/
uint64_t
virtio_read_device_config_8(struct virtio_softc *sc, int index)
{
bus_space_tag_t iot = sc->sc_devcfg_iot;
bus_space_handle_t ioh = sc->sc_devcfg_ioh;
union {
uint64_t u64;
uint32_t l[2];
} v;
uint64_t val;
v.l[0] = bus_space_read_4(iot, ioh, index);
v.l[1] = bus_space_read_4(iot, ioh, index + 4);
if (sc->sc_bus_endian != sc->sc_struct_endian) {
v.l[0] = bswap32(v.l[0]);
v.l[1] = bswap32(v.l[1]);
}
val = v.u64;
if (BYTE_ORDER != sc->sc_struct_endian)
val = bswap64(val);
/*
* In the older virtio spec, device config registers are host endian. On newer
* they are little endian. Some newer devices however explicitly specify their
* register to always be little endian. These functions cater for these.
*/
uint16_t
virtio_read_device_config_le_2(struct virtio_softc *sc, int index)
{
bus_space_tag_t iot = sc->sc_devcfg_iot;
bus_space_handle_t ioh = sc->sc_devcfg_ioh;
uint16_t val;
val = bus_space_read_2(iot, ioh, index);
#if !defined(__aarch64__) && !defined(__arm__)
/*
* For big-endian aarch64/armv7, bus endian is always LSB, but
* byte-order is automatically swapped by bus_space(9) (see also
* comments in virtio_pci.c). Therefore, no need to swap here.
*/
if (sc->sc_bus_endian != LITTLE_ENDIAN)
val = bswap16(val);
#endif
if (BYTE_ORDER != sc->sc_bus_endian)
value = bswap32(value);
bus_space_write_4(iot, ioh, index, value);
}
/*
* The Virtio spec explicitly tells that reading and writing 8 bytes are not
* considered atomic and no triggers may be connected to reading or writing
* it. We access it using two 32 bit writes. For good measure it is stated to
* always write lsb first just in case of a hypervisor bug. See See virtio
* spec 4.1.3.1.
*/
void
virtio_write_device_config_8(struct virtio_softc *sc, int index,
uint64_t value)
{
bus_space_tag_t iot = sc->sc_devcfg_iot;
bus_space_handle_t ioh = sc->sc_devcfg_ioh;
union {
uint64_t u64;
uint32_t l[2];
} v;
if (BYTE_ORDER != sc->sc_struct_endian)
value = bswap64(value);
if (sc->sc_struct_endian == LITTLE_ENDIAN) {
bus_space_write_4(iot, ioh, index, v.l[0]);
bus_space_write_4(iot, ioh, index + 4, v.l[1]);
} else {
bus_space_write_4(iot, ioh, index + 4, v.l[1]);
bus_space_write_4(iot, ioh, index, v.l[0]);
}
}
/*
* In the older virtio spec, device config registers are host endian. On newer
* they are little endian. Some newer devices however explicitly specify their
* register to always be little endian. These functions cater for these.
*/
void
virtio_write_device_config_le_2(struct virtio_softc *sc, int index,
uint16_t value)
{
bus_space_tag_t iot = sc->sc_devcfg_iot;
bus_space_handle_t ioh = sc->sc_devcfg_ioh;
if (sc->sc_bus_endian != LITTLE_ENDIAN)
value = bswap16(value);
bus_space_write_2(iot, ioh, index, value);
}
/*
* Increase the event index in order to delay interrupts.
*/
int
virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq,
uint16_t nslots)
{
uint16_t idx, nused;
idx = vq->vq_used_idx + nslots;
/* set the new event index: avail_ring->used_event = idx */
*vq->vq_used_event = virtio_rw16(sc, idx);
vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE);
vq->vq_queued++;
/*
* Postpone interrupt until 3/4 of the available descriptors have been
* consumed.
*/
int
virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq)
{
uint16_t nslots;
/*
* Postpone interrupt until all of the available descriptors have been
* consumed.
*/
int
virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq)
{
uint16_t nslots;
if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) {
/*
* No way to disable the interrupt completely with
* RingEventIdx. Instead advance used_event by half the
* possible value. This won't happen soon and is far enough in
* the past to not trigger a spurious interrupt.
*/
*vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000);
vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE);
} else {
vq->vq_avail->flags |=
virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT);
vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
}
vq->vq_queued++;
}
int
virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
{
if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) {
/*
* If event index feature is negotiated, enabling interrupts
* is done through setting the latest consumed index in the
* used_event field
*/
*vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx);
vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE);
} else {
vq->vq_avail->flags &=
~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT);
vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
}
vq->vq_queued++;
/* device must be already deactivated */
/* confirm the vq is empty */
s = vq->vq_free_idx;
i = 0;
while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) {
s = vq->vq_desc[s].next;
i++;
}
if (i != vq->vq_num) {
printf("%s: freeing non-empty vq, index %d\n",
device_xname(sc->sc_dev), vq->vq_index);
return EBUSY;
}
/* tell device that there's no virtqueue any longer */
sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
/*
* Enqueue several dmamaps as a single request.
*/
/*
* Typical usage:
* <queue size> number of followings are stored in arrays
* - command blocks (in dmamem) should be pre-allocated and mapped
* - dmamaps for command blocks should be pre-allocated and loaded
* - dmamaps for payload should be pre-allocated
* r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot
* if (r) // currently 0 or EAGAIN
* return r;
* r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
* if (r) {
* virtio_enqueue_abort(sc, vq, slot);
* return r;
* }
* r = virtio_enqueue_reserve(sc, vq, slot,
* dmamap_payload[slot]->dm_nsegs + 1);
* // ^ +1 for command
* if (r) { // currently 0 or EAGAIN
* bus_dmamap_unload(dmat, dmamap_payload[slot]);
* return r; // do not call abort()
* }
* <setup and prepare commands>
* bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
* bus_dmamap_sync(dmat, dmamap_payload[slot],...);
* virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
* virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
* virtio_enqueue_commit(sc, vq, slot, true);
*/
/*
* enqueue_prep: allocate a slot number
*/
int
virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
{
uint16_t slot;
notify:
if (notifynow) {
uint16_t o, n, t;
uint16_t flags;
o = virtio_rw16(sc, vq->vq_avail->idx) - 1;
n = vq->vq_avail_idx;
/*
* Prepare for `device->CPU' (host->guest) transfer
* into the buffer. This must happen before we commit
* the vq->vq_avail->idx update to ensure we're not
* still using the buffer in case program-prior loads
* or stores in it get delayed past the store to
* vq->vq_avail->idx.
*/
vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD);
/* ensure payload is published, then avail idx */
vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE);
vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx);
vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
vq->vq_queued++;
/*
* Dequeue a request.
*/
/*
* dequeue: dequeue a request from uring; dmamap_sync for uring is
* already done in the interrupt handler.
*/
int
virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
int *slotp, int *lenp)
{
uint16_t slot, usedidx;
if (vq->vq_descx[slot].use_indirect)
vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
if (slotp)
*slotp = slot;
if (lenp)
*lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len);
return 0;
}
/*
* dequeue_commit: complete dequeue; the slot is recycled for future use.
* if you forget to call this the slot will be leaked.
*/
int
virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
{
struct vring_desc_extra *vdx;
