/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2015-2018 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
#include <linux/arm-smccc.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/assembler.h>
#include <asm/cpufeature.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/mmu.h>
#include <asm/spectre.h>
.macro save_caller_saved_regs_vect
/* x0 and x1 were saved in the vector entry */
stp x2, x3, [sp, #-16]
!
stp x4, x5, [sp, #-16]
!
stp x6, x7, [sp, #-16]
!
stp x8, x9, [sp, #-16]
!
stp x10, x11, [sp, #-16]
!
stp x12, x13, [sp, #-16]
!
stp x14, x15, [sp, #-16]
!
stp x16, x17, [sp, #-16]
!
.endm
.macro restore_caller_saved_regs_vect
ldp x16, x17, [sp], #16
ldp x14, x15, [sp], #16
ldp x12, x13, [sp], #16
ldp x10, x11, [sp], #16
ldp x8, x9, [sp], #16
ldp x6, x7, [sp], #16
ldp x4, x5, [sp], #16
ldp x2, x3, [sp], #16
ldp x0, x1, [sp], #16
.endm
.text
el1_sync: // Guest trapped into EL2
mrs x0, esr_el2
ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH
cmp x0, #ESR_ELx_EC_HVC64
ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
b.ne el1_trap
/*
* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
* The workaround has already been applied on the host,
* so let's quickly get back to the guest. We don't bother
* restoring x1, as it can be clobbered anyway.
*/
ldr x1, [sp] // Guest
's x0
eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
cbz w1, wa_epilogue
/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
ARM_SMCCC_ARCH_WORKAROUND_2)
cbz w1, wa_epilogue
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
ARM_SMCCC_ARCH_WORKAROUND_3)
cbnz w1, el1_trap
wa_epilogue:
mov x0, xzr
add sp, sp, #16
eret
sb
el1_trap:
get_vcpu_ptr x1, x0
mov x0, #ARM_EXCEPTION_TRAP
b __guest_exit
el1_irq:
el1_fiq:
get_vcpu_ptr x1, x0
mov x0, #ARM_EXCEPTION_IRQ
b __guest_exit
el1_error:
get_vcpu_ptr x1, x0
mov x0, #ARM_EXCEPTION_EL1_SERROR
b __guest_exit
el2_sync:
/* Check for illegal exception return */
mrs x0, spsr_el2
tbnz x0, #20, 1f
save_caller_saved_regs_vect
stp x29, x30, [sp, #-16]
!
bl kvm_unexpected_el2_exception
ldp x29, x30, [sp], #16
restore_caller_saved_regs_vect
eret
1:
/* Let's attempt a recovery from the illegal exception return */
get_vcpu_ptr x1, x0
mov x0, #ARM_EXCEPTION_IL
b __guest_exit
el2_error:
save_caller_saved_regs_vect
stp x29, x30, [sp, #-16]
!
bl kvm_unexpected_el2_exception
ldp x29, x30, [sp], #16
restore_caller_saved_regs_vect
eret
sb
.macro invalid_vector label, target = __guest_exit_panic
.
align 2
SYM_CODE_START_LOCAL(\label)
b \target
SYM_CODE_END(\label)
.endm
/* None of these should ever happen */
invalid_vector el2t_sync_invalid
invalid_vector el2t_irq_invalid
invalid_vector el2t_fiq_invalid
invalid_vector el2t_error_invalid
invalid_vector el2h_irq_invalid
invalid_vector el2h_fiq_invalid
.ltorg
.
align 11
.macro check_preamble_length start, end
/* kvm_patch_vector_branch() generates code that jumps over the preamble. */
.if ((\end-\start)
!= KVM_VECTOR_PREAMBLE)
.error
"KVM vector preamble length mismatch"
.endif
.endm
.macro valid_vect target
.
align 7
661:
esb
stp x0, x1, [sp, #-16]
!
662:
/*
* spectre vectors __bp_harden_hyp_vecs generate br instructions at runtime
* that jump at offset 8 at __kvm_hyp_vector.
* As hyp .text is guarded section, it needs bti j.
*/
bti j
b \target
check_preamble_length 661b, 662b
.endm
.macro invalid_vect target
.
align 7
661:
nop
stp x0, x1, [sp, #-16]
!
662:
/* Check valid_vect */
bti j
b \target
check_preamble_length 661b, 662b
.endm
SYM_CODE_START(__kvm_hyp_vector)
invalid_vect el2t_sync_invalid // Synchronous EL2t
invalid_vect el2t_irq_invalid // IRQ EL2t
invalid_vect el2t_fiq_invalid // FIQ EL2t
invalid_vect el2t_error_invalid // Error EL2t
valid_vect el2_sync // Synchronous EL2h
invalid_vect el2h_irq_invalid // IRQ EL2h
invalid_vect el2h_fiq_invalid // FIQ EL2h
valid_vect el2_error // Error EL2h
valid_vect el1_sync // Synchronous 64-bit EL1
valid_vect el1_irq // IRQ 64-bit EL1
valid_vect el1_fiq // FIQ 64-bit EL1
valid_vect el1_error // Error 64-bit EL1
valid_vect el1_sync // Synchronous 32-bit EL1
valid_vect el1_irq // IRQ 32-bit EL1
valid_vect el1_fiq // FIQ 32-bit EL1
valid_vect el1_error // Error 32-bit EL1
SYM_CODE_END(__kvm_hyp_vector)
.macro spectrev2_smccc_wa1_smc
sub sp, sp, #(8 * 4)
stp x2, x3, [sp, #(8 * 0)]
stp x0, x1, [sp, #(8 * 2)]
alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3
/* Patched to mov WA3 when supported */
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
alternative_cb_end
smc #0
ldp x2, x3, [sp, #(8 * 0)]
add sp, sp, #(8 * 2)
.endm
.macro hyp_ventry indirect, spectrev2
.
align 7
1: esb
.if \spectrev2
!= 0
spectrev2_smccc_wa1_smc
.else
stp x0, x1, [sp, #-16]
!
mitigate_spectre_bhb_loop x0
mitigate_spectre_bhb_clear_insn
.endif
.if \indirect
!= 0
alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch
/*
* For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with:
*
* movz x0, #(addr & 0xffff)
* movk x0, #((addr >> 16) & 0xffff), lsl #16
* movk x0, #((addr >> 32) & 0xffff), lsl #32
* br x0
*
* Where:
* addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE.
* See kvm_patch_vector_branch for details.
*/
nop
nop
nop
nop
alternative_cb_end
.endif
b __kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE)
.endm
.macro generate_vectors indirect, spectrev2
0:
.rept 16
hyp_ventry \indirect, \spectrev2
.endr
.org 0b + SZ_2K // Safety measure
.endm
.
align 11
SYM_CODE_START(__bp_harden_hyp_vecs)
generate_vectors indirect = 0, spectrev2 = 1 // HYP_VECTOR_SPECTRE_DIRECT
generate_vectors indirect = 1, spectrev2 = 0 // HYP_VECTOR_INDIRECT
generate_vectors indirect = 1, spectrev2 = 1 // HYP_VECTOR_SPECTRE_INDIRECT
1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
.org 1b
SYM_CODE_END(__bp_harden_hyp_vecs)
