/*-
* Copyright (c) 2011 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Matt Thomas of 3am Software Foundry.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
uint32_t page_bitmap = atomic_swap_32(&ti->ti_synci_page_bitmap, 0);
#if 0
printf("%s: need to sync %#x\n", __func__, page_bitmap);
#endif
ti->ti_evcnt_synci_asts.ev_count++;
/*
* If every bit is set in the bitmap, sync the entire icache.
*/
if (page_bitmap == pmap_tlb_synci_map_mask) {
pmap_md_icache_sync_all();
ti->ti_evcnt_synci_all.ev_count++;
ti->ti_evcnt_synci_pages.ev_count += pmap_tlb_synci_page_mask+1;
return;
}
/*
* Loop through the bitmap clearing each set of indices for each page.
*/
for (vaddr_t va = 0;
page_bitmap != 0;
page_bitmap >>= 1, va += PAGE_SIZE) {
if (page_bitmap & 1) {
/*
* Each bit set represents a page index to be synced.
*/
pmap_md_icache_sync_range_index(va, PAGE_SIZE);
ti->ti_evcnt_synci_pages.ev_count++;
}
}
}
void
pmap_tlb_syncicache(vaddr_t va, const kcpuset_t *page_onproc)
{
KASSERT(kpreempt_disabled());
/*
* We don't sync the icache here but let ast do it for us just before
* returning to userspace. We do this because we don't really know
* on which CPU we will return to userspace and if we synch the icache
* now it might not be on the CPU we need it on. In addition, others
* threads might sync the icache before we get to return to userland
* so there's no reason for us to do it.
*
* Each TLB/cache keeps a synci sequence number which gets advanced
* each time that TLB/cache performs a pmap_md_sync_icache_all. When
* we return to userland, we check the pmap's corresponding synci
* sequence number for that TLB/cache. If they match, it means that
* no one has yet synched the icache so we much do it ourselves. If
* they don't match someone has already synced the icache for us.
*
* There is a small chance that the generation numbers will wrap and
* then become equal but that's a one in 4 billion cache and will
* just cause an extra sync of the icache.
*/
struct cpu_info * const ci = curcpu();
kcpuset_t *onproc;
kcpuset_create(&onproc, true);
const uint32_t page_mask =
1L << ((va >> PGSHIFT) & pmap_tlb_synci_page_mask);
for (size_t i = 0; i < pmap_ntlbs; i++) {
struct pmap_tlb_info * const ti = pmap_tlbs[i];
TLBINFO_LOCK(ti);
for (;;) {
uint32_t old_page_bitmap = ti->ti_synci_page_bitmap;
if (old_page_bitmap & page_mask) {
ti->ti_evcnt_synci_duplicate.ev_count++;
break;
}
if (orig_page_bitmap == old_page_bitmap) {
if (old_page_bitmap == 0) {
kcpuset_merge(onproc, ti->ti_kcpuset);
} else {
ti->ti_evcnt_synci_deferred.ev_count++;
}
ti->ti_evcnt_synci_desired.ev_count++;
break;
}
}
#if 0
printf("%s: %s: %x to %x on cpus %#x\n", __func__,
ti->ti_name, page_mask, ti->ti_synci_page_bitmap,
onproc & page_onproc & ti->ti_cpu_mask);
#endif
TLBINFO_UNLOCK(ti);
}
kcpuset_intersect(onproc, page_onproc);
if (__predict_false(!kcpuset_iszero(onproc))) {
/*
* If the cpu need to sync this page, tell the current lwp
* to sync the icache before it returns to userspace.
*/
if (kcpuset_isset(onproc, cpu_index(ci))) {
if (ci->ci_flags & CPUF_USERPMAP) {
curlwp->l_md.md_astpending = 1; /* force call to ast() */
ci->ci_evcnt_synci_onproc_rqst.ev_count++;
} else {
ci->ci_evcnt_synci_deferred_rqst.ev_count++;
}
kcpuset_clear(onproc, cpu_index(ci));
}
/*
* For each cpu that is affect, send an IPI telling
* that CPU that the current thread needs to sync its icache.
* We might cause some spurious icache syncs but that's not
* going to break anything.
*/
for (cpuid_t n = kcpuset_ffs(onproc);
n-- > 0;
n = kcpuset_ffs(onproc)) {
kcpuset_clear(onproc, n);
cpu_send_ipi(cpu_lookup(n), IPI_SYNCICACHE);
}
}
kcpuset_destroy(onproc);
}
/*
* We might have been notified because another CPU changed an exec
* page and now needs us to sync the icache so tell the current lwp
* to do the next time it returns to userland (which should be very
* soon).
*/
if (ti->ti_synci_page_bitmap && (ci->ci_flags & CPUF_USERPMAP)) {
curlwp->l_md.md_astpending = 1; /* force call to ast() */
ci->ci_evcnt_synci_ipi_rqst.ev_count++;
}
