/* * scv enters with MSR[EE]=1 and is immediately considered soft-masked. * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED, * and interrupts may be masked and pending already. * system_call_exception() will call trace_hardirqs_off() which means * interrupts could already have been blocked before trace_hardirqs_off, * but this is the best we can do.
*/
/* * Zero user registers to prevent influencing speculative execution * state of kernel code.
*/
SANITIZE_SYSCALL_GPRS()
bl CFUNC(system_call_exception)
.Lsyscall_vectored_\name\()_exit:
addi r4,r1,STACK_INT_FRAME_REGS
li r5,1 /* scv */
bl CFUNC(syscall_exit_prepare)
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
.Lsyscall_vectored_\name\()_rst_start:
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- syscall_vectored_\name\()_restart
li r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
li r11,0
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
ld r2,_CCR(r1) ld r4,_NIP(r1) ld r5,_MSR(r1)
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
/* rfscv returns with LR->NIA and CTR->MSR */
mtlr r4
mtctr r5
/* Could zero these as per ABI, but we may consider a stricter ABI * which preserves these if libc implementations can benefit, so
* restore them for now until further measurement is done. */
REST_GPR(0, r1)
REST_GPRS(4, 8, r1) /* Zero volatile regs that may contain sensitive kernel data */
ZEROIZE_GPRS(9, 12)
mtspr SPRN_XER,r0
/* * We don't need to restore AMR on the way back to userspace for KUAP. * The value of AMR only matters while we're in the kernel.
*/
mtcr r2
REST_GPRS(2, 3, r1)
REST_GPR(13, r1)
REST_GPR(1, r1)
RFSCV_TO_USER
b . /* prevent speculative execution */
/* * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0 * which is tested by system_call_exception when r0 is -1 (as set by vector * entry code).
*/
system_call_vectored sigill 0x7ff0
#endif /* CONFIG_PPC_BOOK3S */
.balign IFETCH_ALIGN_BYTES
.globl system_call_common_real
system_call_common_real:
_ASM_NOKPROBE_SYMBOL(system_call_common_real) ld r10,PACAKMSR(r13) /* get MSR value for kernel */
mtmsrd r10
.balign IFETCH_ALIGN_BYTES
.globl system_call_common
system_call_common:
_ASM_NOKPROBE_SYMBOL(system_call_common)
mr r10,r1 ld r1,PACAKSAVE(r13)
std r10,0(r1)
std r11,_NIP(r1)
std r12,_MSR(r1)
std r0,GPR0(r1)
std r10,GPR1(r1)
std r2,GPR2(r1)
#ifdef CONFIG_PPC_E500
START_BTB_FLUSH_SECTION
BTB_FLUSH(r10)
END_BTB_FLUSH_SECTION
#endif
LOAD_PACA_TOC()
mfcr r12
li r11,0 /* Save syscall parameters in r3-r8 */
SAVE_GPRS(3, 8, r1) /* Zero r9-r12, this should only be required when restoring all GPRs */
std r11,GPR9(r1)
std r11,GPR10(r1)
std r11,GPR11(r1)
std r11,GPR12(r1)
std r9,GPR13(r1)
SAVE_NVGPRS(r1)
std r11,_XER(r1)
std r11,_CTR(r1)
mflr r10
/* * This clears CR0.SO (bit 28), which is the error indication on * return from this system call.
*/
rldimi r12,r11,28,(63-28)
li r11,0xc00
std r10,_LINK(r1)
std r11,_TRAP(r1)
std r12,_CCR(r1)
std r3,ORIG_GPR3(r1)
LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
std r11,STACK_INT_FRAME_MARKER(r1) /* "regs" marker */ /* Calling convention has r3 = regs, r4 = orig r0 */
addi r3,r1,STACK_INT_FRAME_REGS
mr r4,r0
#ifdef CONFIG_PPC_BOOK3S
li r11,1
stb r11,PACASRR_VALID(r13)
#endif
/* * We always enter kernel from userspace with irq soft-mask enabled and * nothing pending. system_call_exception() will call * trace_hardirqs_off().
*/
li r11,IRQS_ALL_DISABLED
stb r11,PACAIRQSOFTMASK(r13)
#ifdef CONFIG_PPC_BOOK3S
li r12,-1 /* Set MSR_EE and MSR_RI */
mtmsrd r12,1
#else
wrteei 1
#endif
/* * Zero user registers to prevent influencing speculative execution * state of kernel code.
*/
SANITIZE_SYSCALL_GPRS()
bl CFUNC(system_call_exception)
.Lsyscall_exit:
addi r4,r1,STACK_INT_FRAME_REGS
li r5,0 /* !scv */
bl CFUNC(syscall_exit_prepare)
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Lsyscall_rst_start:
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- syscall_restart
#endif
li r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
li r11,0
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
/* * We don't need to restore AMR on the way back to userspace for KUAP. * The value of AMR only matters while we're in the kernel.
*/
mtcr r2
REST_GPRS(2, 3, r1)
REST_GPR(13, r1)
REST_GPR(1, r1)
RFI_TO_USER
b . /* prevent speculative execution */
/* * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not * touched, no exit work created, then this can be used.
*/
.balign IFETCH_ALIGN_BYTES
.globl fast_interrupt_return_srr
fast_interrupt_return_srr:
_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
kuap_check_amr r3, r4 ld r5,_MSR(r1)
andi. r0,r5,MSR_PR
#ifdef CONFIG_PPC_BOOK3S
beq 1f
kuap_user_restore r3, r4
b .Lfast_user_interrupt_return_srr
1: kuap_kernel_restore r3, r4
andi. r0,r5,MSR_RI
li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
bne+ .Lfast_kernel_interrupt_return_srr
addi r3,r1,STACK_INT_FRAME_REGS
bl CFUNC(unrecoverable_exception)
b . /* should not get here */
#else
bne .Lfast_user_interrupt_return_srr
b .Lfast_kernel_interrupt_return_srr
#endif
.macro interrupt_return_macro srr
.balign IFETCH_ALIGN_BYTES
.globl interrupt_return_\srr
interrupt_return_\srr\():
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()) ld r4,_MSR(r1)
andi. r0,r4,MSR_PR
beq interrupt_return_\srr\()_kernel
interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
addi r3,r1,STACK_INT_FRAME_REGS
bl CFUNC(interrupt_exit_user_prepare)
#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
cmpdi r3,0
bne- .Lrestore_nvgprs_\srr
.Lrestore_nvgprs_\srr\()_cont:
#endif
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Linterrupt_return_\srr\()_user_rst_start:
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- interrupt_return_\srr\()_user_restart
#endif
li r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
li r11,0
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
.Linterrupt_return_\srr\()_kernel_rst_start: ld r11,SOFTE(r1)
cmpwi r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
beq .Linterrupt_return_\srr\()_soft_enabled
/* * Returning to soft-disabled context. * Check if a MUST_HARD_MASK interrupt has become pending, in which * case we need to disable MSR[EE] in the return context. * * The MSR[EE] check catches among other things the short incoherency * in hard_irq_disable() between clearing MSR[EE] and setting * PACA_IRQ_HARD_DIS.
*/ ld r12,_MSR(r1)
andi. r10,r12,MSR_EE
beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled
lbz r11,PACAIRQHAPPENED(r13)
andi. r10,r11,PACA_IRQ_MUST_HARD_MASK
bne 1f // HARD_MASK is pending
// No HARD_MASK pending, clear possible HARD_DIS set by interrupt
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
stb r11,PACAIRQHAPPENED(r13)
b .Lfast_kernel_interrupt_return_\srr\()
1: /* Must clear MSR_EE from _MSR */
#ifdef CONFIG_PPC_BOOK3S
li r10,0 /* Clear valid before changing _MSR */
.ifc \srr,srr
stb r10,PACASRR_VALID(r13)
.else
stb r10,PACAHSRR_VALID(r13)
.endif
#endif
xori r12,r12,MSR_EE
std r12,_MSR(r1)
b .Lfast_kernel_interrupt_return_\srr\()
.Linterrupt_return_\srr\()_soft_enabled: /* * In the soft-enabled case, need to double-check that we have no * pending interrupts that might have come in before we reached the * restart section of code, and restart the exit so those can be * handled. * * If there are none, it is be possible that the interrupt still * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the * interrupted context. This clear will not clobber a new pending * interrupt coming in, because we're in the restart section, so * such would return to the restart location.
*/
#ifdef CONFIG_PPC_BOOK3S
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- interrupt_return_\srr\()_kernel_restart
#endif
li r11,0
stb r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
FTR_SECTION_ELSE
ldarx r0,0,r1
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
ld r3,_LINK(r1) ld r4,_CTR(r1) ld r5,_XER(r1) ld r6,_CCR(r1)
li r0,0
REST_GPRS(7, 12, r1)
mtlr r3
mtctr r4
mtspr SPRN_XER,r5
/* * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse * the reliable stack unwinder later on. Clear it.
*/
std r0,STACK_INT_FRAME_MARKER(r1)
1: /* * Emulate stack store with update. New r1 value was already calculated * and updated in our interrupt regs by emulate_loadstore, but we can't * store the previous value of r1 to the stack before re-loading our * registers from it, otherwise they could be clobbered. Use * PACA_EXGEN as temporary storage to hold the store data, as * interrupts are disabled here so it won't be clobbered.
*/
mtcr r6
std r9,PACA_EXGEN+0(r13)
addi r9,r1,INT_FRAME_SIZE /* get original r1 */
REST_GPR(6, r1)
REST_GPR(0, r1)
REST_GPR(1, r1)
std r9,0(r1) /* perform store component of stdu */ ld r9,PACA_EXGEN+0(r13)
#ifdef CONFIG_PPC_BOOK3S
_GLOBAL(ret_from_fork_scv)
bl CFUNC(schedule_tail)
HANDLER_RESTORE_NVGPRS()
li r3,0 /* fork() return value */
b .Lsyscall_vectored_common_exit
#endif
_GLOBAL(ret_from_fork)
bl CFUNC(schedule_tail)
HANDLER_RESTORE_NVGPRS()
li r3,0 /* fork() return value */
b .Lsyscall_exit
_GLOBAL(ret_from_kernel_user_thread)
bl CFUNC(schedule_tail)
mtctr r14
mr r3,r15
#ifdef CONFIG_PPC64_ELF_ABI_V2
mr r12,r14
#endif
bctrl
li r3,0 /* * It does not matter whether this returns via the scv or sc path * because it returns as execve() and therefore has no calling ABI * (i.e., it sets registers according to the exec()ed entry point).
*/
b .Lsyscall_exit
_GLOBAL(start_kernel_thread)
bl CFUNC(schedule_tail)
mtctr r14
mr r3,r15
#ifdef CONFIG_PPC64_ELF_ABI_V2
mr r12,r14
#endif
bctrl /* * This must not return. We actually want to BUG here, not WARN, * because BUG will exit the process which is what the kernel thread * should have done, which may give some hope of continuing.
*/
100: trap
EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
Messung V0.5
¤ Dauer der Verarbeitung: 0.16 Sekunden
(vorverarbeitet)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
