struct kvm_resize_hpt { /* These fields read-only after init */ struct kvm *kvm; struct work_struct work;
u32 order;
/* These fields protected by kvm->arch.mmu_setup_lock */
/* Possible values and their usage: * <0 an error occurred during allocation, * -EBUSY allocation is in the progress, * 0 allocation made successfully.
*/ int error;
/* Private to the work thread, until error != -EBUSY, * then protected by kvm->arch.mmu_setup_lock.
*/ struct kvm_hpt_info hpt;
};
int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)
{ int err = -EBUSY; struct kvm_hpt_info info;
mutex_lock(&kvm->arch.mmu_setup_lock); if (kvm->arch.mmu_ready) {
kvm->arch.mmu_ready = 0; /* order mmu_ready vs. vcpus_running */
smp_mb(); if (atomic_read(&kvm->arch.vcpus_running)) {
kvm->arch.mmu_ready = 1; goto out;
}
} if (kvm_is_radix(kvm)) {
err = kvmppc_switch_mmu_to_hpt(kvm); if (err) goto out;
}
if (kvm->arch.hpt.order == order) { /* We already have a suitable HPT */
/* Set the entire HPT to 0, i.e. invalid HPTEs */
memset((void *)kvm->arch.hpt.virt, 0, 1ul << order); /* * Reset all the reverse-mapping chains for all memslots
*/
kvmppc_rmap_reset(kvm);
err = 0; goto out;
}
if (kvm->arch.hpt.virt) {
kvmppc_free_hpt(&kvm->arch.hpt);
kvmppc_rmap_reset(kvm);
}
for (i = 0; i < npages; ++i) {
addr = i << porder; /* can't use hpt_hash since va > 64 bits */
hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25)))
& kvmppc_hpt_mask(&kvm->arch.hpt); /* * We assume that the hash table is empty and no * vcpus are using it at this stage. Since we create * at most one HPTE per HPTEG, we just assume entry 7 * is available and use it.
*/
hash = (hash << 3) + 7;
hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
hp_r = hp1 | addr;
ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
&idx_ret); if (ret != H_SUCCESS) {
pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
addr, ret); break;
}
}
}
int kvmppc_mmu_hv_init(void)
{ unsignedlong nr_lpids;
if (!mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE)) return -EINVAL;
/* Storage key permission check for POWER7 */ if (data && virtmode) { int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr); if (amrfield & 1)
gpte->may_read = 0; if (amrfield & 2)
gpte->may_write = 0;
}
/* Get the guest physical address */
gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr); return 0;
}
/* * Quick test for whether an instruction is a load or a store. * If the instruction is a load or a store, then this will indicate * which it is, at least on server processors. (Embedded processors * have some external PID instructions that don't follow the rule * embodied here.) If the instruction isn't a load or store, then * this doesn't return anything useful.
*/ staticint instruction_is_store(ppc_inst_t instr)
{ unsignedint mask; unsignedint suffix;
int kvmppc_hv_emulate_mmio(struct kvm_vcpu *vcpu, unsignedlong gpa, gva_t ea, int is_store)
{
ppc_inst_t last_inst; bool is_prefixed = !!(kvmppc_get_msr(vcpu) & SRR1_PREFIXED);
/* * Fast path - check if the guest physical address corresponds to a * device on the FAST_MMIO_BUS, if so we can avoid loading the * instruction all together, then we can just handle it and return.
*/ if (is_store) { int idx, ret;
/* * If we fail, we just return to the guest and try executing it again.
*/ if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
EMULATE_DONE) return RESUME_GUEST;
/* * WARNING: We do not know for sure whether the instruction we just * read from memory is the same that caused the fault in the first * place. * * If the fault is prefixed but the instruction is not or vice * versa, try again so that we don't advance pc the wrong amount.
*/ if (ppc_inst_prefixed(last_inst) != is_prefixed) return RESUME_GUEST;
/* * If the instruction we read is neither an load or a store, * then it can't access memory, so we don't need to worry about * enforcing access permissions. So, assuming it is a load or * store, we just check that its direction (load or store) is * consistent with the original fault, since that's what we * checked the access permissions against. If there is a mismatch * we just return and retry the instruction.
*/
if (instruction_is_store(last_inst) != !!is_store) return RESUME_GUEST;
/* * Emulated accesses are emulated by looking at the hash for * translation once, then performing the access later. The * translation could be invalidated in the meantime in which * point performing the subsequent memory access on the old * physical address could possibly be a security hole for the * guest (but not the host). * * This is less of an issue for MMIO stores since they aren't * globally visible. It could be an issue for MMIO loads to * a certain extent but we'll ignore it for now.
*/
if (kvm_is_radix(kvm)) return kvmppc_book3s_radix_page_fault(vcpu, ea, dsisr);
/* * Real-mode code has already searched the HPT and found the * entry we're interested in. Lock the entry and check that * it hasn't changed. If it has, just return and re-execute the * instruction.
*/ if (ea != vcpu->arch.pgfault_addr) return RESUME_GUEST;
/* No memslot means it's an emulated MMIO region */ if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) return kvmppc_hv_emulate_mmio(vcpu, gpa, ea,
dsisr & DSISR_ISSTORE);
/* * This should never happen, because of the slot_is_aligned() * check in kvmppc_do_h_enter().
*/ if (gfn_base < memslot->base_gfn) return -EFAULT;
/* used to check for invalidations in progress */
mmu_seq = kvm->mmu_invalidate_seq;
smp_rmb();
ret = -EFAULT;
page = NULL;
writing = (dsisr & DSISR_ISSTORE) != 0; /* If writing != 0, then the HPTE must allow writing, if we get here */
write_ok = writing;
hva = gfn_to_hva_memslot(memslot, gfn);
/* * Read the PTE from the process' radix tree and use that * so we get the shift and attribute bits.
*/
spin_lock(&kvm->mmu_lock);
ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
pte = __pte(0); if (ptep)
pte = READ_ONCE(*ptep);
spin_unlock(&kvm->mmu_lock); /* * If the PTE disappeared temporarily due to a THP * collapse, just return and let the guest try again.
*/ if (!pte_present(pte)) { if (page)
put_page(page); return RESUME_GUEST;
}
hpa = pte_pfn(pte) << PAGE_SHIFT;
pte_size = PAGE_SIZE; if (shift)
pte_size = 1ul << shift;
is_ci = pte_ci(pte);
if (psize > pte_size) goto out_put; if (pte_size > psize)
hpa |= hva & (pte_size - psize);
/* Check WIMG vs. the actual page we're accessing */ if (!hpte_cache_flags_ok(r, is_ci)) { if (is_ci) goto out_put; /* * Allow guest to map emulated device memory as * uncacheable, but actually make it cacheable.
*/
r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
}
/* * Set the HPTE to point to hpa. * Since the hpa is at PAGE_SIZE granularity, make sure we * don't mask out lower-order bits if psize < PAGE_SIZE.
*/ if (psize < PAGE_SIZE)
psize = PAGE_SIZE;
r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) | hpa; if (hpte_is_writable(r) && !write_ok)
r = hpte_make_readonly(r);
ret = RESUME_GUEST;
preempt_disable(); while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
cpu_relax();
hnow_v = be64_to_cpu(hptep[0]);
hnow_r = be64_to_cpu(hptep[1]); if (cpu_has_feature(CPU_FTR_ARCH_300)) {
hnow_v = hpte_new_to_old_v(hnow_v, hnow_r);
hnow_r = hpte_new_to_old_r(hnow_r);
}
/* * If the HPT is being resized, don't update the HPTE, * instead let the guest retry after the resize operation is complete. * The synchronization for mmu_ready test vs. set is provided * by the HPTE lock.
*/ if (!kvm->arch.mmu_ready) goto out_unlock;
if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] ||
rev->guest_rpte != hpte[2]) /* HPTE has been changed under us; let the guest retry */ goto out_unlock;
hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
/* Always put the HPTE in the rmap chain for the page base address */
rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
lock_rmap(rmap);
/* Check if we might have been invalidated; let the guest retry if so */
ret = RESUME_GUEST; if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
unlock_rmap(rmap); goto out_unlock;
}
/* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) { /* HPTE was previously valid, so we need to invalidate it */
unlock_rmap(rmap);
hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
kvmppc_invalidate_hpte(kvm, hptep, index); /* don't lose previous R and C bits */
r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
} else {
kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
}
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
r = hpte_old_to_new_r(hpte[0], r);
hpte[0] = hpte_old_to_new_v(hpte[0]);
}
hptep[1] = cpu_to_be64(r);
eieio();
__unlock_hpte(hptep, hpte[0]); asmvolatile("ptesync" : : : "memory");
preempt_enable(); if (page && hpte_is_writable(r))
set_page_dirty_lock(page);
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn]; for (;;) {
lock_rmap(rmapp); if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
unlock_rmap(rmapp); break;
}
/* * To avoid an ABBA deadlock with the HPTE lock bit, * we can't spin on the HPTE lock while holding the * rmap chain lock.
*/
i = *rmapp & KVMPPC_RMAP_INDEX;
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4)); if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax(); continue;
}
kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
unlock_rmap(rmapp);
__unlock_hpte(hptep, be64_to_cpu(hptep[0]));
}
}
for (n = memslot->npages; n; --n, ++gfn) { /* * Testing the present bit without locking is OK because * the memslot has been marked invalid already, and hence * no new HPTEs referencing this page can be created, * thus the present bit can't go from 0 to 1.
*/ if (*rmapp & KVMPPC_RMAP_PRESENT)
kvm_unmap_rmapp(kvm, memslot, gfn);
++rmapp;
}
}
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
retry:
lock_rmap(rmapp); if (*rmapp & KVMPPC_RMAP_REFERENCED) {
*rmapp &= ~KVMPPC_RMAP_REFERENCED;
ret = true;
} if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
unlock_rmap(rmapp); return ret;
}
i = head = *rmapp & KVMPPC_RMAP_INDEX; do {
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
j = rev[i].forw;
/* If this HPTE isn't referenced, ignore it */ if (!(be64_to_cpu(hptep[1]) & HPTE_R_R)) continue;
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax(); goto retry;
}
/* Now check and modify the HPTE */ if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
(be64_to_cpu(hptep[1]) & HPTE_R_R)) {
kvmppc_clear_ref_hpte(kvm, hptep, i); if (!(rev[i].guest_rpte & HPTE_R_R)) {
rev[i].guest_rpte |= HPTE_R_R;
note_hpte_modification(kvm, &rev[i]);
}
ret = true;
}
__unlock_hpte(hptep, be64_to_cpu(hptep[0]));
} while ((i = j) != head);
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn]; if (*rmapp & KVMPPC_RMAP_REFERENCED) returntrue;
lock_rmap(rmapp); if (*rmapp & KVMPPC_RMAP_REFERENCED) goto out;
if (*rmapp & KVMPPC_RMAP_PRESENT) {
i = head = *rmapp & KVMPPC_RMAP_INDEX; do {
hp = (unsignedlong *)(kvm->arch.hpt.virt + (i << 4));
j = rev[i].forw; if (be64_to_cpu(hp[1]) & HPTE_R_R) goto out;
} while ((i = j) != head);
}
ret = false;
/* * Returns the number of system pages that are dirty. * This can be more than 1 if we find a huge-page HPTE.
*/ staticint kvm_test_clear_dirty_npages(struct kvm *kvm, unsignedlong *rmapp)
{ struct revmap_entry *rev = kvm->arch.hpt.rev; unsignedlong head, i, j; unsignedlong n; unsignedlong v, r;
__be64 *hptep; int npages_dirty = 0;
i = head = *rmapp & KVMPPC_RMAP_INDEX; do { unsignedlong hptep1;
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
j = rev[i].forw;
/* * Checking the C (changed) bit here is racy since there * is no guarantee about when the hardware writes it back. * If the HPTE is not writable then it is stable since the * page can't be written to, and we would have done a tlbie * (which forces the hardware to complete any writeback) * when making the HPTE read-only. * If vcpus are running then this call is racy anyway * since the page could get dirtied subsequently, so we * expect there to be a further call which would pick up * any delayed C bit writeback. * Otherwise we need to do the tlbie even if C==0 in * order to pick up any delayed writeback of C.
*/
hptep1 = be64_to_cpu(hptep[1]); if (!(hptep1 & HPTE_R_C) &&
(!hpte_is_writable(hptep1) || vcpus_running(kvm))) continue;
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
cpu_relax(); goto retry;
}
/* Now check and modify the HPTE */ if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
__unlock_hpte(hptep, be64_to_cpu(hptep[0])); continue;
}
/* need to make it temporarily absent so C is stable */
hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
kvmppc_invalidate_hpte(kvm, hptep, i);
v = be64_to_cpu(hptep[0]);
r = be64_to_cpu(hptep[1]); if (r & HPTE_R_C) {
hptep[1] = cpu_to_be64(r & ~HPTE_R_C); if (!(rev[i].guest_rpte & HPTE_R_C)) {
rev[i].guest_rpte |= HPTE_R_C;
note_hpte_modification(kvm, &rev[i]);
}
n = kvmppc_actual_pgsz(v, r);
n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT; if (n > npages_dirty)
npages_dirty = n;
eieio();
}
v &= ~HPTE_V_ABSENT;
v |= HPTE_V_VALID;
__unlock_hpte(hptep, v);
} while ((i = j) != head);
preempt_disable();
rmapp = memslot->arch.rmap; for (i = 0; i < memslot->npages; ++i) { int npages = kvm_test_clear_dirty_npages(kvm, rmapp); /* * Note that if npages > 0 then i must be a multiple of npages, * since we always put huge-page HPTEs in the rmap chain * corresponding to their page base address.
*/ if (npages)
set_dirty_bits(map, i, npages);
++rmapp;
}
preempt_enable(); return 0;
}
/* We need to mark this page dirty in the memslot dirty_bitmap, if any */
gfn = gpa >> PAGE_SHIFT;
srcu_idx = srcu_read_lock(&kvm->srcu);
memslot = gfn_to_memslot(kvm, gfn); if (memslot && memslot->dirty_bitmap)
set_bit_le(gfn - memslot->base_gfn, memslot->dirty_bitmap);
srcu_read_unlock(&kvm->srcu, srcu_idx);
}
/* Guest is stopped, so new HPTEs can't be added or faulted * in, only unmapped or altered by host actions. So, it's
* safe to check this before we take the HPTE lock */
vpte = be64_to_cpu(hptep[0]); if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT)) return 0; /* nothing to do */
while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
cpu_relax();
vpte = be64_to_cpu(hptep[0]);
ret = 0; if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT)) /* Nothing to do */ goto out;
if (!(vpte & HPTE_V_1TB_SEG)) { unsignedlong offset, vsid;
/* We only have 28 - 23 bits of offset in avpn */
offset = (avpn & 0x1f) << 23;
vsid = avpn >> 5; /* We can find more bits from the pteg value */ if (pshift < 23)
offset |= ((vsid ^ pteg) & old_hash_mask) << pshift;
if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
BUG_ON(new->order >= old->order);
if (replace_vpte & HPTE_V_BOLTED) { if (vpte & HPTE_V_BOLTED) /* Bolted collision, nothing we can do */
ret = -ENOSPC; /* Discard the new HPTE */ goto out;
}
new_hptep[1] = cpu_to_be64(rpte);
new->rev[new_idx].guest_rpte = guest_rpte; /* No need for a barrier, since new HPT isn't active */
new_hptep[0] = cpu_to_be64(vpte);
unlock_hpte(new_hptep, vpte);
/* Request is still current? */ if (kvm->arch.resize_hpt == resize) { /* We may request large allocations here: * do not sleep with kvm->arch.mmu_setup_lock held for a while.
*/
mutex_unlock(&kvm->arch.mmu_setup_lock);
resize_hpt_debug(resize, "%s(): order = %d\n", __func__,
resize->order);
err = resize_hpt_allocate(resize);
/* We have strict assumption about -EBUSY * when preparing for HPT resize.
*/ if (WARN_ON(err == -EBUSY))
err = -EINPROGRESS;
mutex_lock(&kvm->arch.mmu_setup_lock); /* It is possible that kvm->arch.resize_hpt != resize * after we grab kvm->arch.mmu_setup_lock again.
*/
}
resize->error = err;
if (kvm->arch.resize_hpt != resize)
resize_hpt_release(kvm, resize);
if (resize) { if (resize->order == shift) { /* Suitable resize in progress? */
ret = resize->error; if (ret == -EBUSY)
ret = 100; /* estimated time in ms */ elseif (ret)
resize_hpt_release(kvm, resize);
goto out;
}
/* not suitable, cancel it */
resize_hpt_release(kvm, resize);
}
ret = 0; if (!shift) goto out; /* nothing to do */
/* start new resize */
resize = kzalloc(sizeof(*resize), GFP_KERNEL); if (!resize) {
ret = -ENOMEM; goto out;
}
/* This shouldn't be possible */
ret = -EIO; if (WARN_ON(!kvm->arch.mmu_ready)) goto out_no_hpt;
/* Stop VCPUs from running while we mess with the HPT */
kvm->arch.mmu_ready = 0;
smp_mb();
/* Boot all CPUs out of the guest so they re-read
* mmu_ready */
on_each_cpu(resize_hpt_boot_vcpu, NULL, 1);
ret = -ENXIO; if (!resize || (resize->order != shift)) goto out;
ret = resize->error; if (ret) goto out;
ret = resize_hpt_rehash(resize); if (ret) goto out;
resize_hpt_pivot(resize);
out: /* Let VCPUs run again */
kvm->arch.mmu_ready = 1;
smp_mb();
out_no_hpt:
resize_hpt_release(kvm, resize);
mutex_unlock(&kvm->arch.mmu_setup_lock); return ret;
}
/* * Functions for reading and writing the hash table via reads and * writes on a file descriptor. * * Reads return the guest view of the hash table, which has to be * pieced together from the real hash table and the guest_rpte * values in the revmap array. * * On writes, each HPTE written is considered in turn, and if it * is valid, it is written to the HPT as if an H_ENTER with the * exact flag set was done. When the invalid count is non-zero * in the header written to the stream, the kernel will make * sure that that many HPTEs are invalid, and invalidate them * if not.
*/
/* * Returns 1 if this HPT entry has been modified or has pending * R/C bit changes.
*/ staticint hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
{ unsignedlong rcbits_unset;
if (revp->guest_rpte & HPTE_GR_MODIFIED) return 1;
/* Also need to consider changes in reference and changed bits */
rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C); if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
(be64_to_cpu(hptp[1]) & rcbits_unset)) return 1;
return 0;
}
staticlong record_hpte(unsignedlong flags, __be64 *hptp, unsignedlong *hpte, struct revmap_entry *revp, int want_valid, int first_pass)
{ unsignedlong v, r, hr; unsignedlong rcbits_unset; int ok = 1; int valid, dirty;
/* Unmodified entries are uninteresting except on the first pass */
dirty = hpte_dirty(revp, hptp); if (!first_pass && !dirty) return 0;
v = r = 0; if (valid || dirty) { /* lock the HPTE so it's stable and read it */
preempt_disable(); while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
cpu_relax();
v = be64_to_cpu(hptp[0]);
hr = be64_to_cpu(hptp[1]); if (cpu_has_feature(CPU_FTR_ARCH_300)) {
v = hpte_new_to_old_v(v, hr);
hr = hpte_new_to_old_r(hr);
}
/* re-evaluate valid and dirty from synchronized HPTE value */
valid = !!(v & HPTE_V_VALID);
dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
/* Harvest R and C into guest view if necessary */
rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C); if (valid && (rcbits_unset & hr)) {
revp->guest_rpte |= (hr &
(HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
dirty = 1;
}
if (v & HPTE_V_ABSENT) {
v &= ~HPTE_V_ABSENT;
v |= HPTE_V_VALID;
valid = 1;
} if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
valid = 0;
r = revp->guest_rpte; /* only clear modified if this is the right sort of entry */ if (valid == want_valid && dirty) {
r &= ~HPTE_GR_MODIFIED;
revp->guest_rpte = r;
}
unlock_hpte(hptp, be64_to_cpu(hptp[0]));
preempt_enable(); if (!(valid == want_valid && (first_pass || dirty)))
ok = 0;
}
hpte[0] = cpu_to_be64(v);
hpte[1] = cpu_to_be64(r); return ok;
}
if (rwflag == O_RDONLY) {
mutex_lock(&kvm->slots_lock);
atomic_inc(&kvm->arch.hpte_mod_interest); /* make sure kvmppc_do_h_enter etc. see the increment */
synchronize_srcu_expedited(&kvm->srcu);
mutex_unlock(&kvm->slots_lock);
}
return ret;
}
struct debugfs_htab_state { struct kvm *kvm; struct mutex mutex; unsignedlong hpt_index; int chars_left; int buf_index; char buf[64];
};
ret = mutex_lock_interruptible(&p->mutex); if (ret) return ret;
if (p->chars_left) {
n = p->chars_left; if (n > len)
n = len;
r = copy_to_user(buf, p->buf + p->buf_index, n);
n -= r;
p->chars_left -= n;
p->buf_index += n;
buf += n;
len -= n;
ret = n; if (r) { if (!n)
ret = -EFAULT; goto out;
}
}
i = p->hpt_index;
hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE)); for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);
++i, hptp += 2) { if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))) continue;
/* lock the HPTE so it's stable and read it */
preempt_disable(); while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
cpu_relax();
v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
hr = be64_to_cpu(hptp[1]);
gr = kvm->arch.hpt.rev[i].guest_rpte;
unlock_hpte(hptp, v);
preempt_enable();
if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT))) continue;
n = scnprintf(p->buf, sizeof(p->buf), "%6lx %.16lx %.16lx %.16lx\n",
i, v, hr, gr);
p->chars_left = n; if (n > len)
n = len;
r = copy_to_user(buf, p->buf, n);
n -= r;
p->chars_left -= n;
p->buf_index = n;
buf += n;
len -= n;
ret += n; if (r) { if (!ret)
ret = -EFAULT; goto out;
}
}
p->hpt_index = i;
