/* Invoked from TLB miss handler, we are in the * MMU global registers and they are setup like * this: * * %g1: TSB entry pointer * %g2: available temporary * %g3: FAULT_CODE_{D,I}TLB * %g4: available temporary * %g5: available temporary * %g6: TAG TARGET * %g7: available temporary, will be loaded by us with * the physical address base of the linux page * tables for the current address space
*/
tsb_miss_dtlb:
mov TLB_TAG_ACCESS, %g4
ldxa [%g4] ASI_DMMU, %g4 srlx %g4, PAGE_SHIFT, %g4
ba,pt %xcc, tsb_miss_page_table_walk sllx %g4, PAGE_SHIFT, %g4
/* We need an aligned pair of registers containing 2 values * which can be easily rematerialized. %g6 and %g7 foot the * bill just nicely. We'll save %g6 away into %g2 for the * huge page TSB TAG comparison. * * Perform a huge page TSB lookup.
*/
mov %g6, %g2
and %g5, 0x7, %g6
mov 512, %g7
andn %g5, 0x7, %g5 sllx %g7, %g6, %g7 srlx %g4, REAL_HPAGE_SHIFT, %g6 sub %g7, 1, %g7
and %g6, %g7, %g6 sllx %g6, 4, %g6
add %g5, %g6, %g5
/* No match, remember the huge page TSB entry address, * and restore %g6 and %g7.
*/
TRAP_LOAD_TRAP_BLOCK(%g7, %g6) srlx %g4, 22, %g6
80: stx %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP]
#endif
ldx [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7
/* At this point we have: * %g1 -- TSB entry address * %g3 -- FAULT_CODE_{D,I}TLB * %g4 -- missing virtual address * %g6 -- TAG TARGET (vaddr >> 22) * %g7 -- page table physical address * * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE * TSB both lack a matching entry.
*/
tsb_miss_page_table_walk_sun4v_fastpath:
USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
/* It is a huge page, use huge page TSB entry address we * calculated above. If the huge page TSB has not been * allocated, setup a trap stack and call hugetlb_setup() * to do so, then return from the trap to replay the TLB * miss. * * This is necessary to handle the case of transparent huge * pages where we don't really have a non-atomic context * in which to allocate the hugepage TSB hash table. When * the 'mm' faults in the hugepage for the first time, we * thus handle it here. This also makes sure that we can * allocate the TSB hash table on the correct NUMA node.
*/
TRAP_LOAD_TRAP_BLOCK(%g7, %g2) ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g1
cmp %g1, -1
bne,pt %xcc, 60f
nop
/* For sun4v the ASI_DTLB_DATA_IN store and the retry * instruction get nop'd out and we get here to branch * to the sun4v tlb load code. The registers are setup * as follows: * * %g4: vaddr * %g5: PTE * %g6: TAG * * The sun4v TLB load wants the PTE in %g3 so we fix that * up here.
*/
ba,pt %xcc, sun4v_dtlb_load
mov %g5, %g3
tsb_itlb_load: /* Executable bit must be set. */
661: sethi %hi(_PAGE_EXEC_4U), %g4
andcc %g5, %g4, %g0
.section .sun4v_2insn_patch, "ax"
.word 661b
andcc %g5, _PAGE_EXEC_4V, %g0
nop
.previous
/* For sun4v the ASI_ITLB_DATA_IN store and the retry * instruction get nop'd out and we get here to branch * to the sun4v tlb load code. The registers are setup * as follows: * * %g4: vaddr * %g5: PTE * %g6: TAG * * The sun4v TLB load wants the PTE in %g3 so we fix that * up here.
*/
ba,pt %xcc, sun4v_itlb_load
mov %g5, %g3
/* No valid entry in the page tables, do full fault * processing.
*/
/* Reload MMU related context switch state at * schedule() time. * * %o0: page table physical address * %o1: TSB base config pointer * %o2: TSB huge config pointer, or NULL if none * %o3: Hypervisor TSB descriptor physical address * %o4: Secondary context to load, if non-zero * * We have to run this whole thing with interrupts * disabled so that the current cpu doesn't change * due to preemption.
*/
.align 32
.globl __tsb_context_switch
.type __tsb_context_switch,#function
__tsb_context_switch:
rdpr %pstate, %g1
wrpr %g1, PSTATE_IE, %pstate
/* This can definitely be computed faster... */ srlx %o0, 4, %o5 /* Build index */
and %o5, 511, %o5 /* Mask index */ sllx %o5, %g1, %o5 /* Put into vaddr position */
or %o4, %o5, %o4 /* Full VADDR. */ srlx %o4, %g1, %o4 /* Shift down to create index */
and %o4, %o3, %o4 /* Mask with new_tsb_nents-1 */ sllx %o4, 4, %o4 /* Shift back up into tsb ent offset */
TSB_STORE(%o2 + %o4, %g2) /* Store TAG */
add %o4, 0x8, %o4 /* Advance to TTE */
TSB_STORE(%o2 + %o4, %g3) /* Store TTE */
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
