/*
* Copyright (c) 1999, 2000 Matthew R. Green
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Copyright (c) 2001, 2002 Eduardo Horvath
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* UltraSPARC IOMMU support; used by both the sbus and pci code.
*/
/*
* Setup the iommu.
*
* The sun4u iommu is part of the SBUS or PCI controller so we will
* deal with it here..
*
* For sysio and psycho/psycho+ the IOMMU address space always ends at
* 0xffffe000, but the starting address depends on the size of the
* map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
* entry is 8 bytes. The start of the map can be calculated by
* (0xffffe000 << (8 + is->is_tsbsize)).
*
* But sabre and hummingbird use a different scheme that seems to
* be hard-wired, so we read the start and size from the PROM and
* just use those values.
*/
if (strncmp(name, "pyro", 4) == 0) {
is->is_cr = IOMMUREG_READ(is, iommu_cr);
is->is_cr &= ~IOMMUCR_FIRE_BE;
is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
IOMMUCR_FIRE_TE);
} else
is->is_cr = IOMMUCR_EN;
is->is_tsbsize = tsbsize;
if (iovabase == -1) {
is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
is->is_dvmaend = IOTSB_VEND - 1;
} else {
is->is_dvmabase = iovabase;
is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
}
/*
* Allocate memory for I/O pagetables. They need to be physically
* contiguous.
*/
size = PAGE_SIZE << is->is_tsbsize;
if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
(paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0)
panic("iommu_init: no memory");
va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
if (va == 0)
panic("iommu_init: no memory");
is->is_tsb = (int64_t *)va;
/* Map the pages */
TAILQ_FOREACH(pg, &pglist, pageq.queue) {
pa = VM_PAGE_TO_PHYS(pg);
pmap_kenter_pa(va, pa | PMAP_NVC,
VM_PROT_READ | VM_PROT_WRITE, 0);
va += PAGE_SIZE;
}
pmap_update(pmap_kernel());
memset(is->is_tsb, 0, size);
#ifdef DEBUG
if (iommudebug & IDB_INFO)
{
/* Probe the iommu */
if (!CPU_ISSUN4V) {
printf("iommu cr=%llx tsb=%llx\n",
(unsigned long long)bus_space_read_8(is->is_bustag,
is->is_iommu,
offsetof(struct iommureg, iommu_cr)),
(unsigned long long)bus_space_read_8(is->is_bustag,
is->is_iommu,
offsetof(struct iommureg, iommu_tsb)));
printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
(unsigned long long)is->is_ptsb);
delay(1000000); /* 1 s */
}
}
#endif
/*
* Now all the hardware's working we need to allocate a dvma map.
*/
aprint_debug("DVMA map: %x to %x\n",
(unsigned int)is->is_dvmabase,
(unsigned int)is->is_dvmaend);
aprint_debug("IOTSB: %llx to %llx\n",
(unsigned long long)is->is_ptsb,
(unsigned long long)(is->is_ptsb + size - 1));
is->is_dvmamap = vmem_create(name,
is->is_dvmabase,
(is->is_dvmaend + 1) - is->is_dvmabase,
PAGE_SIZE, /* quantum */
NULL, /* importfn */
NULL, /* releasefn */
NULL, /* source */
0, /* qcache_max */
VM_SLEEP,
IPL_VM);
KASSERT(is->is_dvmamap != NULL);
/*
* Set the TSB size. The relevant bits were moved to the TSB
* base register in the PCIe host bridges.
*/
if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB)
is->is_ptsb |= is->is_tsbsize;
else
is->is_cr |= (is->is_tsbsize << 16);
/*
* now actually start up the IOMMU
*/
iommu_reset(is);
}
/*
* Streaming buffers don't exist on the UltraSPARC IIi; we should have
* detected that already and disabled them. If not, we will notice that
* they aren't there when the STRBUF_EN bit does not remain.
*/
void
iommu_reset(struct iommu_state *is)
{
int i;
struct strbuf_ctl *sb;
if (CPU_ISSUN4V)
return;
IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
/* Enable IOMMU in diagnostic mode */
IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE);
for (i = 0; i < 2; i++) {
if ((sb = is->is_sb[i])) {
/* No streaming buffers? Disable them */
if (bus_space_read_8(is->is_bustag,
is->is_sb[i]->sb_sb,
STRBUFREG(strbuf_ctl)) == 0) {
is->is_sb[i]->sb_flush = NULL;
} else {
/*
* locate the pa of the flush buffer.
*/
if (pmap_extract(pmap_kernel(),
(vaddr_t)is->is_sb[i]->sb_flush,
&is->is_sb[i]->sb_flushpa) == FALSE)
is->is_sb[i]->sb_flush = NULL;
}
}
}
if (is->is_flags & IOMMU_FLUSH_CACHE)
IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
}
/*
* Here are the iommu control routines.
*/
static void
iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
{
DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx flags %x\n",
va, (long)pa, flags));
if (!CPU_ISSUN4V)
iommu_enter_sun4u(sb, va, pa, flags);
else
iommu_enter_sun4v(sb, va, pa, flags);
}
void
iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
{
struct iommu_state *is = sb->sb_is;
int strbuf = (flags & BUS_DMA_STREAMING);
int64_t tte;
#ifdef DIAGNOSTIC
if (va < is->is_dvmabase || va > is->is_dvmaend)
panic("iommu_enter: va %#lx not in DVMA space", va);
#endif
/* Is the streamcache flush really needed? */
if (sb->sb_flush)
iommu_strbuf_flush(sb, va);
else
/* If we can't flush the strbuf don't enable it. */
strbuf = 0;
/*
* iommu_remove: removes mappings created by iommu_enter
*
* Only demap from IOMMU if flag is set.
*
* XXX: this function needs better internal error checking.
*/
static void
iommu_remove(struct iommu_state *is, vaddr_t va, size_t len)
{
DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx len %zu\n", va, len));
if (!CPU_ISSUN4V)
iommu_remove_sun4u(is, va, len);
else
iommu_remove_sun4v(is, va, len);
}
void
iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len)
{
int slot;
#ifdef DIAGNOSTIC
if (va < is->is_dvmabase || va > is->is_dvmaend)
panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
if ((long)(va + len) < (long)va)
panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
(long) va, (long) len);
if (len & ~0xfffffff)
panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
#endif
va = trunc_page(va);
DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
while (len > 0) {
DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
"for va %p size %lx\n",
(int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
(u_long)len));
if (len <= PAGE_SIZE)
len = 0;
else
len -= PAGE_SIZE;
#if 0
/*
* XXX Zero-ing the entry would not require RMW
*
* Disabling valid bit while a page is used by a device
* causes an uncorrectable DMA error.
* Workaround to avoid an uncorrectable DMA error is
* eliminating the next line, but the page is mapped
* until the next iommu_enter call.
*/
is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
membar_StoreStore();
#endif
IOMMUREG_WRITE(is, iommu_flush, va);
/*
* Streaming buffer flushes:
*
* 1 Tell strbuf to flush by storing va to strbuf_pgflush. If
* we're not on a cache line boundary (64-bits):
* 2 Store 0 in flag
* 3 Store pointer to flag in flushsync
* 4 wait till flushsync becomes 0x1
*
* If it takes more than .5 sec, something
* went wrong.
*/
/*
* A boundary presented to bus_dmamem_alloc() takes precedence
* over boundary in the map.
*/
if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
boundary = map->_dm_boundary;
align = uimax(map->dm_segs[0]._ds_align, PAGE_SIZE);
/*
* If our segment size is larger than the boundary we need to
* split the transfer up int little pieces ourselves.
*/
KASSERT(is->is_dvmamap != NULL);
err = vmem_xalloc(is->is_dvmamap, sgsize,
align, /* alignment */
0, /* phase */
(sgsize > boundary) ? 0 : boundary,
VMEM_ADDR_MIN, /* minaddr */
VMEM_ADDR_MAX, /* maxaddr */
VM_NOSLEEP | VM_BESTFIT,
&dvmaddr);
int
iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
{
struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
struct iommu_state *is = sb->sb_is;
struct vm_page *pg;
int i, j;
int left;
int err, needsflush;
bus_size_t sgsize;
paddr_t pa;
bus_size_t boundary, align;
u_long dvmaddr, sgstart, sgend, bmask;
struct pglist *pglist;
const int pagesz = PAGE_SIZE;
int slot;
#ifdef DEBUG
int npg = 0;
#endif
if (map->dm_nsegs) {
/* Already in use?? */
#ifdef DIAGNOSTIC
printf("iommu_dvmamap_load_raw: map still in use\n");
#endif
bus_dmamap_unload(t, map);
}
/*
* A boundary presented to bus_dmamem_alloc() takes precedence
* over boundary in the map.
*/
if ((boundary = segs[0]._ds_boundary) == 0)
boundary = map->_dm_boundary;
align = uimax(segs[0]._ds_align, pagesz);
/*
* Make sure that on error condition we return "no valid mappings".
*/
map->dm_nsegs = 0;
/* Count up the total number of pages we need */
pa = trunc_page(segs[0].ds_addr);
sgsize = 0;
left = size;
for (i = 0; left > 0 && i < nsegs; i++) {
if (round_page(pa) != round_page(segs[i].ds_addr))
sgsize = round_page(sgsize) +
(segs[i].ds_addr & PGOFSET);
sgsize += uimin(left, segs[i].ds_len);
left -= segs[i].ds_len;
pa = segs[i].ds_addr + segs[i].ds_len;
}
sgsize = round_page(sgsize);
/*
* If our segment size is larger than the boundary we need to
* split the transfer up into little pieces ourselves.
*/
const vm_flag_t vmflags = VM_BESTFIT |
((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);
/* Set the active DVMA map */
map->_dm_dvmastart = dvmaddr;
map->_dm_dvmasize = sgsize;
bmask = ~(boundary - 1);
if ((pglist = segs[0]._ds_mlist) == NULL) {
u_long prev_va = 0UL, last_va = dvmaddr;
paddr_t prev_pa = 0;
int end = 0, offset;
bus_size_t len = size;
/*
* This segs is made up of individual physical
* segments, probably by _bus_dmamap_load_uio() or
* _bus_dmamap_load_mbuf(). Ignore the mlist and
* load each one individually.
*/
j = 0;
needsflush = 0;
for (i = 0; i < nsegs ; i++) {
pa = segs[i].ds_addr;
offset = (pa & PGOFSET);
pa = trunc_page(pa);
dvmaddr = trunc_page(dvmaddr);
left = uimin(len, segs[i].ds_len);
if (sgsize == 0)
panic("iommu_dmamap_load_raw: size botch");
/* Now map a series of pages. */
while (dvmaddr <= sgend) {
DPRINTF(IDB_BUSDMA,
("iommu_dvmamap_load_raw: map %p "
"loading va %lx at pa %lx\n",
map, (long)dvmaddr,
(long)(pa)));
/* Enter it if we haven't before. */
if (prev_va != dvmaddr) {
iommu_enter(sb, prev_va = dvmaddr,
prev_pa = pa,
flags | IOTTE_DEBUG(++npg << 12));
needsflush = 1;
/*
* Flush an individual dma segment, returns non-zero if the streaming buffers
* need flushing afterwards.
*/
static int
iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
{
vaddr_t vaend;
struct iommu_state *is = sb->sb_is;
#ifdef DIAGNOSTIC
if (va < is->is_dvmabase || va > is->is_dvmaend)
panic("invalid va: %llx", (long long)va);
#endif
if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
DPRINTF(IDB_SYNC,
("iommu_dvmamap_sync_range: attempting to flush "
"non-streaming entry\n"));
return (0);
}
vaend = round_page(va + len) - 1;
va = trunc_page(va);
#ifdef DIAGNOSTIC
if (va < is->is_dvmabase || vaend > is->is_dvmaend)
panic("invalid va range: %llx to %llx (%x to %x)",
(long long)va, (long long)vaend,
is->is_dvmabase,
is->is_dvmaend);
#endif
for ( ; va <= vaend; va += PAGE_SIZE) {
DPRINTF(IDB_SYNC,
("iommu_dvmamap_sync_range: flushing va %p\n",
(void *)(u_long)va));
iommu_strbuf_flush(sb, va);
}
return (1);
}
static void
_iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
bus_size_t len, int ops)
{
struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
bus_size_t count;
int i, needsflush = 0;
if (!sb->sb_flush)
return;
for (i = 0; i < map->dm_nsegs; i++) {
if (offset < map->dm_segs[i].ds_len)
break;
offset -= map->dm_segs[i].ds_len;
}
if (i == map->dm_nsegs)
panic("%s: segment too short %llu", __func__,
(unsigned long long)offset);
if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
/* Nothing to do */;
}
if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) {
for (; len > 0 && i < map->dm_nsegs; i++) {
count = MIN(map->dm_segs[i].ds_len - offset, len);
if (count > 0 &&
iommu_dvmamap_sync_range(sb,
map->dm_segs[i].ds_addr + offset, count))
needsflush = 1;
offset = 0;
len -= count;
}
#ifdef DIAGNOSTIC
if (i == map->dm_nsegs && len > 0)
panic("%s: leftover %llu", __func__,
(unsigned long long)len);
#endif
/* If len is 0, then there is nothing to do */
if (len == 0)
return;
if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) {
/* Flush the CPU then the IOMMU */
bus_dmamap_sync(t->_parent, map, offset, len, ops);
_iommu_dvmamap_sync(t, map, offset, len, ops);
}
if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) {
/* Flush the IOMMU then the CPU */
_iommu_dvmamap_sync(t, map, offset, len, ops);
bus_dmamap_sync(t->_parent, map, offset, len, ops);
}
}
int
iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
int flags)
{
/*
* Allocate some space in the kernel map, and then map these pages
* into this space.
*/
size = round_page(size);
va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
if (va == 0)
return (ENOMEM);
*kvap = (void *)va;
/*
* digest flags:
*/
cbit = 0;
if (flags & BUS_DMA_COHERENT) /* Disable vcache */
cbit |= PMAP_NVC;
if (flags & BUS_DMA_NOCACHE) /* side effects */
cbit |= PMAP_NC;
/*
* Now take this and map it into the CPU.
*/
pglist = segs[0]._ds_mlist;
TAILQ_FOREACH(pg, pglist, pageq.queue) {
#ifdef DIAGNOSTIC
if (size == 0)
panic("iommu_dvmamem_map: size botch");
#endif
addr = VM_PAGE_TO_PHYS(pg);
DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
"mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
pmap_kenter_pa(va, addr | cbit,
VM_PROT_READ | VM_PROT_WRITE, 0);
va += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(pmap_kernel());
return (0);
}
