/*-
* Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
* NASA Ames Research Center and by Chris G. Demetriou.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Copyright (c) 1992, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* the Systems Programming Group of the University of Utah Computer
* Science Department and Ralph Campbell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)pmap.c 8.4 (Berkeley) 1/26/94
*/
/*
* Manages physical address maps.
*
* In addition to hardware address maps, this
* module is called upon to provide software-use-only
* maps which may or may not be stored in the same
* form as hardware maps. These pseudo-maps are
* used to store intermediate results from copy
* operations to and from address spaces.
*
* Since the information managed by this module is
* also stored by the logical address mapping module,
* this module may throw away valid virtual-to-physical
* mappings at almost any time. However, invalidations
* of virtual-to-physical mappings must be done as
* requested.
*
* In order to cope with hardware architectures which
* make virtual-to-physical map invalidates expensive,
* this module may delay invalidate or reduced protection
* operations until such time as they are actually
* necessary. This module is given full information as
* to which processors are currently using which maps,
* and to when physical maps must be made correct.
*/
#if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \
&& !defined(PMAP_NO_PV_UNCACHED)
#error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \
PMAP_NO_PV_UNCACHED to be defined
#endif
/*
* Update new end.
*/
pmap_curmaxkvaddr = virtual_end;
UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
return virtual_end;
}
/*
* Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()).
* This function allows for early dynamic memory allocation until the virtual
* memory system has been bootstrapped. After that point, either kmem_alloc
* or malloc should be used. This function works by stealing pages from the
* (to be) managed page pool, then implicitly mapping the pages (by using
* their direct mapped addresses) and zeroing them.
*
* It may be used once the physical memory segments have been pre-loaded
* into the vm_physmem[] array. Early memory allocation MUST use this
* interface! This cannot be used after vm_page_startup(), and will
* generate a panic if tried.
*
* Note that this memory will never be freed, and in essence it is wired
* down.
*
* We must adjust *vstartp and/or *vendp iff we use address space
* from the kernel virtual address range defined by pmap_virtual_space().
*/
vaddr_t
pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
{
size_t npgs;
paddr_t pa;
vaddr_t va;
for (uvm_physseg_t bank = uvm_physseg_get_first();
uvm_physseg_valid_p(bank);
bank = uvm_physseg_get_next(bank)) {
if (uvm.page_init_done == true)
panic("pmap_steal_memory: called _after_ bootstrap");
DPRINTF("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n",
__func__, bank,
uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank),
uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank));
if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank)
|| uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) {
DPRINTF("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank);
continue;
}
if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) {
DPRINTF("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n",
__func__, bank, npgs);
continue;
}
if (!pmap_md_ok_to_steal_p(bank, npgs)) {
continue;
}
/*
* Always try to allocate from the segment with the least
* amount of space left.
*/
#define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b)))
if (uvm_physseg_valid_p(maybe_bank) == false
|| VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) {
maybe_bank = bank;
}
}
if (uvm_physseg_valid_p(maybe_bank)) {
const uvm_physseg_t bank = maybe_bank;
/*
* There are enough pages here; steal them!
*/
pa = ptoa(uvm_physseg_get_start(bank));
uvm_physseg_unplug(atop(pa), npgs);
DPRINTF("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n",
__func__, bank, npgs, VM_PHYSMEM_SPACE(bank));
/*
* If we got here, there was no memory left.
*/
panic("pmap_steal_memory: no memory to steal %zu pages", npgs);
}
/*
* Bootstrap the system enough to run with virtual memory.
* (Common routine called by machine-dependent bootstrap code.)
*/
void
pmap_bootstrap_common(void)
{
UVMHIST_LINK_STATIC(pmapexechist);
UVMHIST_LINK_STATIC(pmaphist);
UVMHIST_LINK_STATIC(pmapxtabhist);
/*
* Initialize the segtab lock.
*/
mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH);
pmap_tlb_miss_lock_init();
}
/*
* Initialize the pmap module.
* Called by vm_init, to initialize any structures that the pmap
* system needs to map virtual memory.
*/
void
pmap_init(void)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLED(pmaphist);
/*
* Set a low water mark on the pv_entry pool, so that we are
* more likely to have these around even in extreme memory
* starvation.
*/
pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
/*
* Set the page colormask but allow pmap_md_init to override it.
*/
pmap_page_colormask = ptoa(uvmexp.colormask);
pmap_md_init();
/*
* Now it is safe to enable pv entry recording.
*/
pmap_initialized = true;
}
/*
* Create and return a physical map.
*
* If the size specified for the map
* is zero, the map is an actual physical
* map, and may be referenced by the
* hardware.
*
* If the size specified is non-zero,
* the map will be used in software only, and
* is bounded by that size.
*/
pmap_t
pmap_create(void)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLED(pmaphist);
PMAP_COUNT(create);
/*
* Retire the given physical map from service.
* Should only be called if the map contains
* no valid mappings.
*/
void
pmap_destroy(pmap_t pmap)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0);
UVMHIST_CALLARGS(pmapxtabhist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0);
/*
* Remove this page from all physical maps in which it resides.
* Reflects back modify bits to the pager.
*/
void
pmap_page_remove(struct vm_page_md *mdpg)
{
kpreempt_disable();
VM_PAGEMD_PVLIST_LOCK(mdpg);
pmap_pvlist_check(mdpg);
/* Assume no more - it'll get fixed if there are */
pv->pv_next = NULL;
/*
* pvp is non-null when we already have a PV_KENTER
* pv in pvh_first; otherwise we haven't seen a
* PV_KENTER pv and we need to copy this one to
* pvh_first
*/
if (pvp) {
/*
* The previous PV_KENTER pv needs to point to
* this PV_KENTER pv
*/
pvp->pv_next = pv;
} else {
pv_entry_t fpv = &mdpg->mdpg_first;
*fpv = *pv;
KASSERT(fpv->pv_pmap == pmap_kernel());
}
pvp = pv;
continue;
}
#endif
const pmap_t pmap = pv->pv_pmap;
vaddr_t va = trunc_page(pv->pv_va);
pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
pmap_limits.virtual_end);
pt_entry_t pte = *ptep;
UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx"
" pte %#jx", (uintptr_t)pv, (uintptr_t)pmap, va,
pte_value(pte));
if (!pte_valid_p(pte))
continue;
const bool is_kernel_pmap_p = (pmap == pmap_kernel());
if (is_kernel_pmap_p) {
PMAP_COUNT(remove_kernel_pages);
} else {
PMAP_COUNT(remove_user_pages);
}
if (pte_wired_p(pte))
pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;
pmap_tlb_miss_lock_enter();
const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p);
pte_set(ptep, npte);
if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) {
/*
* Flush the TLB for the given address.
*/
pmap_tlb_invalidate_addr(pmap, va);
}
pmap_tlb_miss_lock_exit();
/*
* non-null means this is a non-pvh_first pv, so we should
* free it.
*/
if (pvp) {
KASSERT(pvp->pv_pmap == pmap_kernel());
KASSERT(pvp->pv_next == NULL);
pmap_pv_free(pv);
} else {
pv->pv_pmap = NULL;
pv->pv_next = NULL;
}
}
#ifdef __HAVE_PMAP_PV_TRACK
/*
* pmap_pv_protect: change protection of an unmanaged pv-tracked page from
* all pmaps that map it
*/
void
pmap_pv_protect(paddr_t pa, vm_prot_t prot)
{
/* the only case is remove at the moment */
KASSERT(prot == VM_PROT_NONE);
struct pmap_page *pp;
pp = pmap_pv_tracked(pa);
if (pp == NULL)
panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR,
pa);
/*
* If pmap_remove_all was called, we deactivated ourselves and nuked
* our ASID. Now we have to reactivate ourselves.
*/
if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) {
pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE;
pmap_tlb_asid_acquire(pmap, curlwp);
pmap_segtab_activate(pmap, curlwp);
}
pmap_tlb_miss_lock_exit();
kpreempt_enable();
KASSERT(kpreempt_disabled());
/*
* Change protection on every valid mapping within this segment.
*/
for (; sva < eva; sva += NBPG, ptep++) {
pt_entry_t pte = *ptep;
if (!pte_valid_p(pte))
continue;
struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
if (pg != NULL && pte_modified_p(pte)) {
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
if (VM_PAGEMD_EXECPAGE_P(mdpg)) {
KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg));
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
if (VM_PAGEMD_CACHED_P(mdpg)) {
#endif
UVMHIST_LOG(pmapexechist,
"pg %#jx (pa %#jx): "
"syncicached performed",
(uintptr_t)pg, VM_PAGE_TO_PHYS(pg),
0, 0);
pmap_page_syncicache(pg);
PMAP_COUNT(exec_synced_protect);
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
}
#endif
}
}
pte = pte_prot_downgrade(pte, prot);
if (*ptep != pte) {
pmap_tlb_miss_lock_enter();
pte_set(ptep, pte);
/*
* Update the TLB if needed.
*/
pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI);
pmap_tlb_miss_lock_exit();
}
}
UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
return false;
}
/*
* Set the physical protection on the
* specified range of this map as requested.
*/
void
pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)",
(uintptr_t)pmap, sva, eva, prot);
PMAP_COUNT(protect);
/*
* Insert the given physical page (p) at
* the specified virtual address (v) in the
* target physical map with the protection requested.
*
* If specified, the page will be wired down, meaning
* that the related pte can not be reclaimed.
*
* NB: This is the only routine which MAY NOT lazy-evaluate
* or lose information. That is, this routine must actually
* insert this page into the given map NOW.
*/
int
pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
const bool wired = (flags & PMAP_WIRED) != 0;
const bool is_kernel_pmap_p = (pmap == pmap_kernel());
#if defined(EFI_RUNTIME)
const bool is_efirt_pmap_p = (pmap == pmap_efirt());
#else
const bool is_efirt_pmap_p = false;
#endif
u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0;
#ifdef UVMHIST
struct kern_history * const histp =
((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist);
#endif
const bool good_color = PMAP_PAGE_COLOROK_P(pa, va);
if (is_kernel_pmap_p) {
PMAP_COUNT(kernel_mappings);
if (!good_color)
PMAP_COUNT(kernel_mappings_bad);
} else {
PMAP_COUNT(user_mappings);
if (!good_color)
PMAP_COUNT(user_mappings_bad);
}
pmap_addr_range_check(pmap, va, va, __func__);
KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x",
VM_PROT_READ, prot);
if (mdpg) {
/* Set page referenced/modified status based on flags */
if (flags & VM_PROT_WRITE) {
pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED);
} else if (flags & VM_PROT_ALL) {
pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED);
}
PMAP_COUNT(managed_mappings);
} else if (mdpp) {
#ifdef __HAVE_PMAP_PV_TRACK
pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED);
PMAP_COUNT(pvtracked_mappings);
#endif
} else if (is_efirt_pmap_p) {
PMAP_COUNT(efirt_mappings);
} else {
/*
* Assumption: if it is not part of our managed memory
* then it must be device memory which may be volatile.
*/
if ((flags & PMAP_CACHE_MASK) == 0)
flags |= PMAP_NOCACHE;
PMAP_COUNT(unmanaged_mappings);
}
KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
pmap_limits.virtual_end);
KASSERT(!pte_valid_p(*ptep));
/*
* No need to track non-managed pages or PMAP_KMPAGEs pages for aliases
*/
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
if (pg != NULL && (flags & PMAP_KMPAGE) == 0
&& pmap_md_virtual_cache_aliasing_p()) {
pmap_enter_pv(pmap, va, pa, mdpg, &npte, PV_KENTER);
}
#endif
/*
* We have the option to force this mapping into the TLB but we
* don't. Instead let the next reference to the page do it.
*/
pmap_tlb_miss_lock_enter();
pte_set(ptep, npte);
pmap_tlb_update_addr(pmap_kernel(), va, npte, 0);
pmap_tlb_miss_lock_exit();
kpreempt_enable();
#if DEBUG > 1
for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) {
if (((long *)va)[i] != ((long *)pa)[i])
panic("%s: contents (%lx) of va %#"PRIxVADDR
" != contents (%lx) of pa %#"PRIxPADDR, __func__,
((long *)va)[i], va, ((long *)pa)[i], pa);
}
#endif
kpreempt_disable();
/*
* Free all of our ASIDs which means we can skip doing all the
* tlb_invalidate_addrs().
*/
pmap_tlb_miss_lock_enter();
#ifdef MULTIPROCESSOR
// This should be the last CPU with this pmap onproc
KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu())));
if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu())))
#endif
pmap_tlb_asid_deactivate(pmap);
#ifdef MULTIPROCESSOR
KASSERT(kcpuset_iszero(pmap->pm_onproc));
#endif
pmap_tlb_asid_release_all(pmap);
pmap_tlb_miss_lock_exit();
pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE;
/*
* Routine: pmap_unwire
* Function: Clear the wired attribute for a map/virtual-address
* pair.
* In/out conditions:
* The mapping must already exist in the pmap.
*/
void
pmap_unwire(pmap_t pmap, vaddr_t va)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va,
0, 0);
PMAP_COUNT(unwire);
/*
* Don't need to flush the TLB since PG_WIRED is only in software.
*/
kpreempt_disable();
pmap_addr_range_check(pmap, va, va, __func__);
pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE",
pmap, va);
pt_entry_t pte = *ptep;
KASSERTMSG(pte_valid_p(pte),
"pmap %p va %#" PRIxVADDR " invalid PTE %#" PRIxPTE " @ %p",
pmap, va, pte_value(pte), ptep);
if (pte_wired_p(pte)) {
pmap_tlb_miss_lock_enter();
pte_set(ptep, pte_unwire_entry(pte));
pmap_tlb_miss_lock_exit();
pmap->pm_stats.wired_count--;
}
#ifdef DIAGNOSTIC
else {
printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n",
__func__, pmap, va);
}
#endif
kpreempt_enable();
UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
}
/*
* Routine: pmap_extract
* Function:
* Extract the physical page address associated
* with the given map/virtual_address pair.
*/
bool
pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
{
paddr_t pa;
if (pmap == pmap_kernel()) {
if (pmap_md_direct_mapped_vaddr_p(va)) {
pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
goto done;
}
if (pmap_md_io_vaddr_p(va))
panic("pmap_extract: io address %#"PRIxVADDR"", va);
/*
* Copy the range specified by src_addr/len
* from the source map to the range dst_addr/len
* in the destination map.
*
* This routine is only advisory and need not do anything.
*/
void
pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len,
vaddr_t src_addr)
{
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "(dpm=#%jx spm=%#jx dva=%#jx sva=%#jx",
(uintptr_t)dst_pmap, (uintptr_t)src_pmap, dst_addr, src_addr);
UVMHIST_LOG(pmaphist, "... len=%#jx)", len, 0, 0, 0);
PMAP_COUNT(copy);
}
/*
* pmap_clear_reference:
*
* Clear the reference bit on the specified physical page.
*/
bool
pmap_clear_reference(struct vm_page *pg)
{
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
/*
* pmap_is_referenced:
*
* Return whether or not the specified physical page is referenced
* by any physical maps.
*/
bool
pmap_is_referenced(struct vm_page *pg)
{
return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg));
}
/*
* Clear the modify bits on the specified physical page.
*/
bool
pmap_clear_modify(struct vm_page *pg)
{
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
pv_entry_t pv = &mdpg->mdpg_first;
pv_entry_t pv_next;
/*
* remove write access from any pages that are dirty
* so we can tell if they are written to again later.
* flush the VAC first if there is one.
*/
kpreempt_disable();
VM_PAGEMD_PVLIST_READLOCK(mdpg);
pmap_pvlist_check(mdpg);
for (; pv != NULL; pv = pv_next) {
pmap_t pmap = pv->pv_pmap;
vaddr_t va = trunc_page(pv->pv_va);
pv_next = pv->pv_next;
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
if (PV_ISKENTER_P(pv))
continue;
#endif
pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
KASSERT(ptep);
pt_entry_t pte = pte_prot_nowrite(*ptep);
if (*ptep == pte) {
continue;
}
KASSERT(pte_valid_p(pte));
const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
pmap_tlb_miss_lock_enter();
pte_set(ptep, pte);
pmap_tlb_invalidate_addr(pmap, va);
pmap_tlb_miss_lock_exit();
pmap_update(pmap);
if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) {
/*
* The list changed! So restart from the beginning.
*/
pv_next = &mdpg->mdpg_first;
pmap_pvlist_check(mdpg);
}
}
pmap_pvlist_check(mdpg);
VM_PAGEMD_PVLIST_UNLOCK(mdpg);
kpreempt_enable();
/*
* pmap_is_modified:
*
* Return whether or not the specified physical page is modified
* by any physical maps.
*/
bool
pmap_is_modified(struct vm_page *pg)
{
return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg));
}
/*
* pmap_set_modified:
*
* Sets the page modified reference bit for the specified page.
*/
void
pmap_set_modified(paddr_t pa)
{
struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED);
}
apv = NULL;
VM_PAGEMD_PVLIST_LOCK(mdpg);
again:
pv = &mdpg->mdpg_first;
pmap_pvlist_check(mdpg);
if (pv->pv_pmap == NULL) {
KASSERT(pv->pv_next == NULL);
/*
* No entries yet, use header as the first entry
*/
PMAP_COUNT(primary_mappings);
PMAP_COUNT(mappings);
#ifdef UVMHIST
first = true;
#endif
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
KASSERT(VM_PAGEMD_CACHED_P(mdpg));
// If the new mapping has an incompatible color the last
// mapping of this page, clean the page before using it.
if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) {
pmap_md_vca_clean(mdpg, PMAP_WBINV);
}
#endif
pv->pv_pmap = pmap;
pv->pv_va = va | flags;
} else {
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
if (pmap_md_vca_add(mdpg, va, nptep)) {
goto again;
}
#endif
/*
* There is at least one other VA mapping this page.
* Place this entry after the header.
*
* Note: the entry may already be in the table if
* we are only changing the protection bits.
*/
for (npv = pv; npv; npv = npv->pv_next) {
if (pmap == npv->pv_pmap
&& va == trunc_page(npv->pv_va)) {
#ifdef PARANOIADIAG
pt_entry_t *ptep = pmap_pte_lookup(pmap, va);
pt_entry_t pte = (ptep != NULL) ? *ptep : 0;
if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa)
printf("%s: found va %#"PRIxVADDR
" pa %#"PRIxPADDR
" in pv_table but != %#"PRIxPTE"\n",
__func__, va, pa, pte_value(pte));
#endif
PMAP_COUNT(remappings);
VM_PAGEMD_PVLIST_UNLOCK(mdpg);
if (__predict_false(apv != NULL))
pmap_pv_free(apv);
UVMHIST_LOG(pmaphist,
" <-- done pv=%#jx (reused)",
(uintptr_t)pv, 0, 0, 0);
return;
}
}
if (__predict_true(apv == NULL)) {
/*
* To allocate a PV, we have to release the PVLIST lock
* so get the page generation. We allocate the PV, and
* then reacquire the lock.
*/
pmap_pvlist_check(mdpg);
const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
apv = (pv_entry_t)pmap_pv_alloc();
if (apv == NULL)
panic("pmap_enter_pv: pmap_pv_alloc() failed");
/*
* If the generation has changed, then someone else
* tinkered with this page so we should start over.
*/
if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg))
goto again;
}
npv = apv;
apv = NULL;
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
/*
* If need to deal with virtual cache aliases, keep mappings
* in the kernel pmap at the head of the list. This allows
* the VCA code to easily use them for cache operations if
* present.
*/
pmap_t kpmap = pmap_kernel();
if (pmap != kpmap) {
while (pv->pv_pmap == kpmap && pv->pv_next != NULL) {
pv = pv->pv_next;
}
}
#endif
npv->pv_va = va | flags;
npv->pv_pmap = pmap;
npv->pv_next = pv->pv_next;
pv->pv_next = npv;
PMAP_COUNT(mappings);
}
pmap_pvlist_check(mdpg);
VM_PAGEMD_PVLIST_UNLOCK(mdpg);
if (__predict_false(apv != NULL))
pmap_pv_free(apv);
/*
* Remove a physical to virtual address translation.
* If cache was inhibited on this page, and there are no more cache
* conflicts, restore caching.
* Flush the cache if the last page is removed (should always be cached
* at this point).
*/
void
pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty)
{
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
pv_entry_t pv, npv;
bool last;
/*
* If it is the first entry on the list, it is actually
* in the header and we must copy the following entry up
* to the header. Otherwise we must search the list for
* the entry. In either case we free the now unused entry.
*/
last = false;
if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) {
npv = pv->pv_next;
if (npv) {
*pv = *npv;
KASSERT(pv->pv_pmap != NULL);
} else {
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
#endif
pv->pv_pmap = NULL;
last = true; /* Last mapping removed */
}
PMAP_COUNT(remove_pvfirst);
} else {
for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) {
PMAP_COUNT(remove_pvsearch);
if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va))
break;
}
if (npv) {
pv->pv_next = npv->pv_next;
}
}
#ifdef PMAP_VIRTUAL_CACHE_ALIASES
pmap_md_vca_remove(pg, va, dirty, last);
#endif
/*
* Free the pv_entry if needed.
*/
if (npv)
pmap_pv_free(npv);
if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) {
if (last) {
/*
* If this was the page's last mapping, we no longer
* care about its execness.
*/
UVMHIST_LOG(pmapexechist,
"pg %#jx (pa %#jx)last %ju: execpage cleared",
(uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0);
pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
PMAP_COUNT(exec_uncached_remove);
} else {
/*
* Someone still has it mapped as an executable page
* so we must sync it.
*/
UVMHIST_LOG(pmapexechist,
"pg %#jx (pa %#jx) last %ju: performed syncicache",
(uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0);
pmap_page_syncicache(pg);
PMAP_COUNT(exec_synced_remove);
}
}
/*
* Allocate a lock on an as-needed basis. This will hopefully give us
* semi-random distribution not based on page color.
*/
if (__predict_false(lock == NULL)) {
size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37);
size_t lockid = locknum & pli->pli_lock_mask;
kmutex_t * const new_lock = pli->pli_locks[lockid];
/*
* Set the lock. If some other thread already did, just use
* the one they assigned.
*/
lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock);
if (lock == NULL) {
lock = new_lock;
atomic_inc_uint(&pli->pli_lock_refs[lockid]);
}
}
/*
* Now finally provide the lock.
*/
return lock;
}
#else /* !MULTIPROCESSOR */
void
pmap_pvlist_lock_init(size_t cache_line_size)
{
mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH);
}
#ifdef MODULAR
kmutex_t *
pmap_pvlist_lock_addr(struct vm_page_md *mdpg)
{
/*
* We just use a global lock.
*/
if (__predict_false(mdpg->mdpg_lock == NULL)) {
mdpg->mdpg_lock = &pmap_pvlist_mutex;
}
/*
* Now finally provide the lock.
*/
return mdpg->mdpg_lock;
}
#endif /* MODULAR */
#endif /* !MULTIPROCESSOR */
/*
* pmap_pv_page_alloc:
*
* Allocate a page for the pv_entry pool.
*/
void *
pmap_pv_page_alloc(struct pool *pp, int flags)
{
struct vm_page * const pg = pmap_md_alloc_poolpage(UVM_PGA_USERESERVE);
if (pg == NULL)
return NULL;
