/* * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. *
*/
// Most of the runtime stubs have this simple frame layout. // This class exists to make the layout shared in one place. // Offsets are for compiler stack slots, which are jints. enum layout { // The frame sender code expects that rbp will be in the "natural" place and // will override any oopMap setting for it. We must therefore force the layout // so that it agrees with the frame sender code.
rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
rbp_off2,
return_off, return_off2,
framesize
};
};
class RegisterSaver { // Capture info about frame layout. Layout offsets are in jint // units because compiler frame slots are jints. #define XSAVE_AREA_BEGIN 160 #define XSAVE_AREA_YMM_BEGIN 576 #define XSAVE_AREA_OPMASK_BEGIN 1088 #define XSAVE_AREA_ZMM_BEGIN 1152 #define XSAVE_AREA_UPPERBANK 1664 #define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off #define DEF_YMM_OFFS(regnum) ymm ## regnum ## _off = ymm_off + (regnum)*16/BytesPerInt, ymm ## regnum ## H_off #define DEF_ZMM_OFFS(regnum) zmm ## regnum ## _off = zmm_off + (regnum)*32/BytesPerInt, zmm ## regnum ## H_off #define DEF_OPMASK_OFFS(regnum) opmask ## regnum ## _off = opmask_off + (regnum)*8/BytesPerInt, opmask ## regnum ## H_off #define DEF_ZMM_UPPER_OFFS(regnum) zmm ## regnum ## _off = zmm_upper_off + (regnum-16)*64/BytesPerInt, zmm ## regnum ## H_off enum layout {
fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt, // fxsave save area
xmm_off = fpu_state_off + XSAVE_AREA_BEGIN/BytesPerInt, // offset in fxsave save area
DEF_XMM_OFFS(0),
DEF_XMM_OFFS(1), // 2..15 are implied in range usage
ymm_off = xmm_off + (XSAVE_AREA_YMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
DEF_YMM_OFFS(0),
DEF_YMM_OFFS(1), // 2..15 are implied in range usage
opmask_off = xmm_off + (XSAVE_AREA_OPMASK_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
DEF_OPMASK_OFFS(0),
DEF_OPMASK_OFFS(1), // 2..7 are implied in range usage
zmm_off = xmm_off + (XSAVE_AREA_ZMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
DEF_ZMM_OFFS(0),
DEF_ZMM_OFFS(1),
zmm_upper_off = xmm_off + (XSAVE_AREA_UPPERBANK - XSAVE_AREA_BEGIN)/BytesPerInt,
DEF_ZMM_UPPER_OFFS(16),
DEF_ZMM_UPPER_OFFS(17), // 18..31 are implied in range usage
fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt),
fpu_stateH_end,
r15_off, r15H_off,
r14_off, r14H_off,
r13_off, r13H_off,
r12_off, r12H_off,
r11_off, r11H_off,
r10_off, r10H_off,
r9_off, r9H_off,
r8_off, r8H_off,
rdi_off, rdiH_off,
rsi_off, rsiH_off,
ignore_off, ignoreH_off, // extra copy of rbp
rsp_off, rspH_off,
rbx_off, rbxH_off,
rdx_off, rdxH_off,
rcx_off, rcxH_off,
rax_off, raxH_off, // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state
align_off, alignH_off,
flags_off, flagsH_off, // The frame sender code expects that rbp will be in the "natural" place and // will override any oopMap setting for it. We must therefore force the layout // so that it agrees with the frame sender code.
rbp_off, rbpH_off, // copy of rbp we will restore
return_off, returnH_off, // slot for return address
reg_save_size // size in compiler stack slots
};
// During deoptimization only the result registers need to be restored, // all the other values have already been extracted. staticvoid restore_result_registers(MacroAssembler* masm);
};
OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors) { int off = 0; int num_xmm_regs = XMMRegister::available_xmm_registers(); #if COMPILER2_OR_JVMCI if (save_wide_vectors && UseAVX == 0) {
save_wide_vectors = false; // vectors larger than 16 byte long are supported only with AVX
}
assert(!save_wide_vectors || MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported"); #else
save_wide_vectors = false; // vectors are generated only by C2 and JVMCI #endif
// Always make the frame size 16-byte aligned, both vector and non vector stacks are always allocated int frame_size_in_bytes = align_up(reg_save_size*BytesPerInt, num_xmm_regs); // OopMap frame size is in compiler stack slots (jint's) not bytes or words int frame_size_in_slots = frame_size_in_bytes / BytesPerInt; // CodeBlob frame size is in words. int frame_size_in_words = frame_size_in_bytes / wordSize;
*total_frame_words = frame_size_in_words;
// Save registers, fpu state, and flags. // We assume caller has already pushed the return address onto the // stack, so rsp is 8-byte aligned here. // We push rpb twice in this sequence because we want the real rbp // to be under the return like a normal enter.
__ enter(); // rsp becomes 16-byte aligned here
__ push_CPU_state(); // Push a multiple of 16 bytes
// push cpu state handles this on EVEX enabled targets if (save_wide_vectors) { // Save upper half of YMM registers(0..15) int base_addr = XSAVE_AREA_YMM_BEGIN; for (int n = 0; n < 16; n++) {
__ vextractf128_high(Address(rsp, base_addr+n*16), as_XMMRegister(n));
} if (VM_Version::supports_evex()) { // Save upper half of ZMM registers(0..15)
base_addr = XSAVE_AREA_ZMM_BEGIN; for (int n = 0; n < 16; n++) {
__ vextractf64x4_high(Address(rsp, base_addr+n*32), as_XMMRegister(n));
} // Save full ZMM registers(16..num_xmm_regs)
base_addr = XSAVE_AREA_UPPERBANK;
off = 0; int vector_len = Assembler::AVX_512bit; for (int n = 16; n < num_xmm_regs; n++) {
__ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);
} #if COMPILER2_OR_JVMCI
base_addr = XSAVE_AREA_OPMASK_BEGIN;
off = 0; for(int n = 0; n < KRegister::number_of_registers; n++) {
__ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n));
} #endif
}
} else { if (VM_Version::supports_evex()) { // Save upper bank of XMM registers(16..31) for scalar or 16-byte vector usage int base_addr = XSAVE_AREA_UPPERBANK;
off = 0; int vector_len = VM_Version::supports_avx512vl() ? Assembler::AVX_128bit : Assembler::AVX_512bit; for (int n = 16; n < num_xmm_regs; n++) {
__ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);
} #if COMPILER2_OR_JVMCI
base_addr = XSAVE_AREA_OPMASK_BEGIN;
off = 0; for(int n = 0; n < KRegister::number_of_registers; n++) {
__ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n));
} #endif
}
}
__ vzeroupper(); if (frame::arg_reg_save_area_bytes != 0) { // Allocate argument register save area
__ subptr(rsp, frame::arg_reg_save_area_bytes);
}
// Set an oopmap for the call site. This oopmap will map all // oop-registers and debug-info registers as callee-saved. This // will allow deoptimization at this safepoint to find all possible // debug-info recordings, as well as let GC find all oops.
OopMapSet *oop_maps = new OopMapSet();
OopMap* map = new OopMap(frame_size_in_slots, 0);
#define STACK_OFFSET(x) VMRegImpl::stack2reg((x))
map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg());
map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg());
map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg());
map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg()); // rbp location is known implicitly by the frame sender code, needs no oopmap // and the location where rbp was saved by is ignored
map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg());
map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r8_off ), r8->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r9_off ), r9->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg());
map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg()); // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15, // on EVEX enabled targets, we get it included in the xsave area
off = xmm0_off; int delta = xmm1_off - off; for (int n = 0; n < 16; n++) {
XMMRegister xmm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());
off += delta;
} if (UseAVX > 2) { // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
off = zmm16_off;
delta = zmm17_off - off; for (int n = 16; n < num_xmm_regs; n++) {
XMMRegister zmm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg());
off += delta;
}
}
#if COMPILER2_OR_JVMCI if (save_wide_vectors) { // Save upper half of YMM registers(0..15)
off = ymm0_off;
delta = ymm1_off - ymm0_off; for (int n = 0; n < 16; n++) {
XMMRegister ymm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), ymm_name->as_VMReg()->next(4));
off += delta;
} if (VM_Version::supports_evex()) { // Save upper half of ZMM registers(0..15)
off = zmm0_off;
delta = zmm1_off - zmm0_off; for (int n = 0; n < 16; n++) {
XMMRegister zmm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next(8));
off += delta;
}
}
} #endif// COMPILER2_OR_JVMCI
// %%% These should all be a waste but we'll keep things as they were for now if (true) {
map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next()); // rbp location is known implicitly by the frame sender code, needs no oopmap
map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r8H_off ), r8->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r9H_off ), r9->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next());
map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next()); // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15, // on EVEX enabled targets, we get it included in the xsave area
off = xmm0H_off;
delta = xmm1H_off - off; for (int n = 0; n < 16; n++) {
XMMRegister xmm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()->next());
off += delta;
} if (UseAVX > 2) { // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
off = zmm16H_off;
delta = zmm17H_off - off; for (int n = 16; n < num_xmm_regs; n++) {
XMMRegister zmm_name = as_XMMRegister(n);
map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next());
off += delta;
}
}
}
return map;
}
void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors) { int num_xmm_regs = XMMRegister::available_xmm_registers(); if (frame::arg_reg_save_area_bytes != 0) { // Pop arg register save area
__ addptr(rsp, frame::arg_reg_save_area_bytes);
}
#if COMPILER2_OR_JVMCI if (restore_wide_vectors) {
assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");
assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
} #else
assert(!restore_wide_vectors, "vectors are generated only by C2"); #endif
__ vzeroupper();
// On EVEX enabled targets everything is handled in pop fpu state if (restore_wide_vectors) { // Restore upper half of YMM registers (0..15) int base_addr = XSAVE_AREA_YMM_BEGIN; for (int n = 0; n < 16; n++) {
__ vinsertf128_high(as_XMMRegister(n), Address(rsp, base_addr+n*16));
} if (VM_Version::supports_evex()) { // Restore upper half of ZMM registers (0..15)
base_addr = XSAVE_AREA_ZMM_BEGIN; for (int n = 0; n < 16; n++) {
__ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, base_addr+n*32));
} // Restore full ZMM registers(16..num_xmm_regs)
base_addr = XSAVE_AREA_UPPERBANK; int vector_len = Assembler::AVX_512bit; int off = 0; for (int n = 16; n < num_xmm_regs; n++) {
__ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);
} #if COMPILER2_OR_JVMCI
base_addr = XSAVE_AREA_OPMASK_BEGIN;
off = 0; for (int n = 0; n < KRegister::number_of_registers; n++) {
__ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8)));
} #endif
}
} else { if (VM_Version::supports_evex()) { // Restore upper bank of XMM registers(16..31) for scalar or 16-byte vector usage int base_addr = XSAVE_AREA_UPPERBANK; int off = 0; int vector_len = VM_Version::supports_avx512vl() ? Assembler::AVX_128bit : Assembler::AVX_512bit; for (int n = 16; n < num_xmm_regs; n++) {
__ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);
} #if COMPILER2_OR_JVMCI
base_addr = XSAVE_AREA_OPMASK_BEGIN;
off = 0; for (int n = 0; n < KRegister::number_of_registers; n++) {
__ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8)));
} #endif
}
}
// Recover CPU state
__ pop_CPU_state(); // Get the rbp described implicitly by the calling convention (no oopMap)
__ pop(rbp);
}
// Just restore result register. Only used by deoptimization. By // now any callee save register that needs to be restored to a c2 // caller of the deoptee has been extracted into the vframeArray // and will be stuffed into the c2i adapter we create for later // restoration so only result registers need to be restored here.
// Pop all of the register save are off the stack except the return address
__ addptr(rsp, return_offset_in_bytes());
}
// Is vector's size (in bytes) bigger than a size saved by default? // 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions. bool SharedRuntime::is_wide_vector(int size) { return size > 16;
}
// --------------------------------------------------------------------------- // Read the array of BasicTypes from a signature, and compute where the // arguments should go. Values in the VMRegPair regs array refer to 4-byte // quantities. Values less than VMRegImpl::stack0 are registers, those above // refer to 4-byte stack slots. All stack slots are based off of the stack pointer // as framesizes are fixed. // VMRegImpl::stack0 refers to the first slot 0(sp). // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. // Register up to Register::number_of_registers are the 64-bit // integer registers.
// Note: the INPUTS in sig_bt are in units of Java argument words, which are // either 32-bit or 64-bit depending on the build. The OUTPUTS are in 32-bit // units regardless of build. Of course for i486 there is no 64 bit build
// The Java calling convention is a "shifted" version of the C ABI. // By skipping the first C ABI register we can call non-static jni methods // with small numbers of arguments without having to shuffle the arguments // at all. Since we control the java ABI we ought to at least get some // advantage out of it.
int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
VMRegPair *regs, int total_args_passed) {
uint int_args = 0;
uint fp_args = 0;
uint stk_args = 0; // inc by 2 each time
for (int i = 0; i < total_args_passed; i++) { switch (sig_bt[i]) { case T_BOOLEAN: case T_CHAR: case T_BYTE: case T_SHORT: case T_INT: if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
} else {
regs[i].set1(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
} break; case T_VOID: // halves of T_LONG or T_DOUBLE
assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
regs[i].set_bad(); break; case T_LONG:
assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); // fall through case T_OBJECT: case T_ARRAY: case T_ADDRESS: if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
} else {
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
} break; case T_FLOAT: if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
} else {
regs[i].set1(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
} break; case T_DOUBLE:
assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
} else {
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
} break; default:
ShouldNotReachHere(); break;
}
}
return align_up(stk_args, 2);
}
// Patch the callers callsite with entry to compiled code if it exists. staticvoid patch_callers_callsite(MacroAssembler *masm) {
Label L;
__ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
__ jcc(Assembler::equal, L);
// Save the current stack pointer
__ mov(r13, rsp); // Schedule the branch target address early. // Call into the VM to patch the caller, then jump to compiled callee // rax isn't live so capture return address while we easily can
__ movptr(rax, Address(rsp, 0));
// align stack so push_CPU_state doesn't fault
__ andptr(rsp, -(StackAlignmentInBytes));
__ push_CPU_state();
__ vzeroupper(); // VM needs caller's callsite // VM needs target method // This needs to be a long call since we will relocate this adapter to // the codeBuffer and it may not reach
// Allocate argument register save area if (frame::arg_reg_save_area_bytes != 0) {
__ subptr(rsp, frame::arg_reg_save_area_bytes);
}
__ mov(c_rarg0, rbx);
__ mov(c_rarg1, rax);
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
// De-allocate argument register save area if (frame::arg_reg_save_area_bytes != 0) {
__ addptr(rsp, frame::arg_reg_save_area_bytes);
}
staticvoid gen_c2i_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, const BasicType *sig_bt, const VMRegPair *regs,
Label& skip_fixup) { // Before we get into the guts of the C2I adapter, see if we should be here // at all. We've come from compiled code and are attempting to jump to the // interpreter, which means the caller made a static call to get here // (vcalls always get a compiled target if there is one). Check for a // compiled target. If there is one, we need to patch the caller's call.
patch_callers_callsite(masm);
__ bind(skip_fixup);
// Since all args are passed on the stack, total_args_passed * // Interpreter::stackElementSize is the space we need.
assert(total_args_passed >= 0, "total_args_passed is %d", total_args_passed);
int extraspace = (total_args_passed * Interpreter::stackElementSize);
// stack is aligned, keep it that way // This is not currently needed or enforced by the interpreter, but // we might as well conform to the ABI.
extraspace = align_up(extraspace, 2*wordSize);
// set senderSP value
__ lea(r13, Address(rsp, wordSize));
#ifdef ASSERT
__ check_stack_alignment(r13, "sender stack not aligned"); #endif if (extraspace > 0) { // Pop the return address
__ pop(rax);
__ subptr(rsp, extraspace);
// Push the return address
__ push(rax);
// Account for the return address location since we store it first rather // than hold it in a register across all the shuffling
extraspace += wordSize;
}
// Now write the args into the outgoing interpreter space for (int i = 0; i < total_args_passed; i++) { if (sig_bt[i] == T_VOID) {
assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half"); continue;
}
// offset to start parameters int st_off = (total_args_passed - i) * Interpreter::stackElementSize; int next_off = st_off - Interpreter::stackElementSize;
// Say 4 args: // i st_off // 0 32 T_LONG // 1 24 T_VOID // 2 16 T_OBJECT // 3 8 T_BOOL // - 0 return address // // However to make thing extra confusing. Because we can fit a long/double in // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter // leaves one slot empty and only stores to a single slot. In this case the // slot that is occupied is the T_VOID slot. See I said it was confusing.
VMReg r_1 = regs[i].first();
VMReg r_2 = regs[i].second(); if (!r_1->is_valid()) {
assert(!r_2->is_valid(), ""); continue;
} if (r_1->is_stack()) { // memory to memory use rax int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace; if (!r_2->is_valid()) { // sign extend??
__ movl(rax, Address(rsp, ld_off));
__ movptr(Address(rsp, st_off), rax);
} else {
__ movq(rax, Address(rsp, ld_off));
// Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG // T_DOUBLE and T_LONG use two slots in the interpreter if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) { // ld_off == LSW, ld_off+wordSize == MSW // st_off == MSW, next_off == LSW
__ movq(Address(rsp, next_off), rax); #ifdef ASSERT // Overwrite the unused slot with known junk
__ mov64(rax, CONST64(0xdeadffffdeadaaaa));
__ movptr(Address(rsp, st_off), rax); #endif/* ASSERT */
} else {
__ movq(Address(rsp, st_off), rax);
}
}
} elseif (r_1->is_Register()) { Register r = r_1->as_Register(); if (!r_2->is_valid()) { // must be only an int (or less ) so move only 32bits to slot // why not sign extend??
__ movl(Address(rsp, st_off), r);
} else { // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG // T_DOUBLE and T_LONG use two slots in the interpreter if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) { // long/double in gpr #ifdef ASSERT // Overwrite the unused slot with known junk
__ mov64(rax, CONST64(0xdeadffffdeadaaab));
__ movptr(Address(rsp, st_off), rax); #endif/* ASSERT */
__ movq(Address(rsp, next_off), r);
} else {
__ movptr(Address(rsp, st_off), r);
}
}
} else {
assert(r_1->is_XMMRegister(), ""); if (!r_2->is_valid()) { // only a float use just part of the slot
__ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
} else { #ifdef ASSERT // Overwrite the unused slot with known junk
__ mov64(rax, CONST64(0xdeadffffdeadaaac));
__ movptr(Address(rsp, st_off), rax); #endif/* ASSERT */
__ movdbl(Address(rsp, next_off), r_1->as_XMMRegister());
}
}
}
void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, const BasicType *sig_bt, const VMRegPair *regs) {
// Note: r13 contains the senderSP on entry. We must preserve it since // we may do a i2c -> c2i transition if we lose a race where compiled // code goes non-entrant while we get args ready. // In addition we use r13 to locate all the interpreter args as // we must align the stack to 16 bytes on an i2c entry else we // lose alignment we expect in all compiled code and register // save code can segv when fxsave instructions find improperly // aligned stack pointer.
// Adapters can be frameless because they do not require the caller // to perform additional cleanup work, such as correcting the stack pointer. // An i2c adapter is frameless because the *caller* frame, which is interpreted, // routinely repairs its own stack pointer (from interpreter_frame_last_sp), // even if a callee has modified the stack pointer. // A c2i adapter is frameless because the *callee* frame, which is interpreted, // routinely repairs its caller's stack pointer (from sender_sp, which is set // up via the senderSP register). // In other words, if *either* the caller or callee is interpreted, we can // get the stack pointer repaired after a call. // This is why c2i and i2c adapters cannot be indefinitely composed. // In particular, if a c2i adapter were to somehow call an i2c adapter, // both caller and callee would be compiled methods, and neither would // clean up the stack pointer changes performed by the two adapters. // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc.
if (VerifyAdapterCalls &&
(Interpreter::code() != NULL || StubRoutines::code1() != NULL)) { // So, let's test for cascading c2i/i2c adapters right now. // assert(Interpreter::contains($return_addr) || // StubRoutines::contains($return_addr), // "i2c adapter must return to an interpreter frame");
__ block_comment("verify_i2c { "); // Pick up the return address
__ movptr(rax, Address(rsp, 0));
Label L_ok; if (Interpreter::code() != NULL)
range_check(masm, rax, r11,
Interpreter::code()->code_start(), Interpreter::code()->code_end(),
L_ok); if (StubRoutines::code1() != NULL)
range_check(masm, rax, r11,
StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
L_ok); if (StubRoutines::code2() != NULL)
range_check(masm, rax, r11,
StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
L_ok); constchar* msg = "i2c adapter must return to an interpreter frame";
__ block_comment(msg);
__ stop(msg);
__ bind(L_ok);
__ block_comment("} verify_i2ce ");
}
// Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code.
__ movptr(r11, rsp);
// Pick up the return address
__ pop(rax);
// Convert 4-byte c2 stack slots to words. int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
if (comp_args_on_stack) {
__ subptr(rsp, comp_words_on_stack * wordSize);
}
// Ensure compiled code always sees stack at proper alignment
__ andptr(rsp, -16);
// push the return address and misalign the stack that youngest frame always sees // as far as the placement of the call instruction
__ push(rax);
// Put saved SP in another register constRegister saved_sp = rax;
__ movptr(saved_sp, r11);
// Will jump to the compiled code just as if compiled code was doing it. // Pre-load the register-jump target early, to schedule it better.
__ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_offset())));
#if INCLUDE_JVMCI if (EnableJVMCI) { // check if this call should be routed towards a specific entry point
__ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
Label no_alternative_target;
__ jcc(Assembler::equal, no_alternative_target);
__ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
__ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
__ bind(no_alternative_target);
} #endif// INCLUDE_JVMCI
// Now generate the shuffle code. Pick up all register args and move the // rest through the floating point stack top. for (int i = 0; i < total_args_passed; i++) { if (sig_bt[i] == T_VOID) { // Longs and doubles are passed in native word order, but misaligned // in the 32-bit build.
assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half"); continue;
}
// Pick up 0, 1 or 2 words from SP+offset.
assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?"); // Load in argument order going down. int ld_off = (total_args_passed - i)*Interpreter::stackElementSize; // Point to interpreter value (vs. tag) int next_off = ld_off - Interpreter::stackElementSize; // // //
VMReg r_1 = regs[i].first();
VMReg r_2 = regs[i].second(); if (!r_1->is_valid()) {
assert(!r_2->is_valid(), ""); continue;
} if (r_1->is_stack()) { // Convert stack slot to an SP offset (+ wordSize to account for return address ) int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
// We can use r13 as a temp here because compiled code doesn't need r13 as an input // and if we end up going thru a c2i because of a miss a reasonable value of r13 // will be generated. if (!r_2->is_valid()) { // sign extend???
__ movl(r13, Address(saved_sp, ld_off));
__ movptr(Address(rsp, st_off), r13);
} else { // // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case // So we must adjust where to pick up the data to match the interpreter. // // Interpreter local[n] == MSW, local[n+1] == LSW however locals // are accessed as negative so LSW is at LOW address
// ld_off is MSW so get LSW constint offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
next_off : ld_off;
__ movq(r13, Address(saved_sp, offset)); // st_off is LSW (i.e. reg.first())
__ movq(Address(rsp, st_off), r13);
}
} elseif (r_1->is_Register()) { // Register argument Register r = r_1->as_Register();
assert(r != rax, "must be different"); if (r_2->is_valid()) { // // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case // So we must adjust where to pick up the data to match the interpreter.
// this can be a misaligned move
__ movq(r, Address(saved_sp, offset));
} else { // sign extend and use a full word?
__ movl(r, Address(saved_sp, ld_off));
}
} else { if (!r_2->is_valid()) {
__ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
} else {
__ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
}
}
}
__ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about
// 6243940 We might end up in handle_wrong_method if // the callee is deoptimized as we race thru here. If that // happens we don't want to take a safepoint because the // caller frame will look interpreted and arguments are now // "compiled" so it is much better to make this transition // invisible to the stack walking code. Unfortunately if // we try and find the callee by normal means a safepoint // is possible. So we stash the desired callee in the thread // and the vm will find there should this case occur.
// put Method* where a c2i would expect should we end up there // only needed because eof c2 resolve stubs return Method* as a result in // rax
__ mov(rax, rbx);
__ jmp(r11);
}
// ------------------------------------------------------------------------- // Generate a C2I adapter. On entry we know rbx holds the Method* during calls // to the interpreter. The args start out packed in the compiled layout. They // need to be unpacked into the interpreter layout. This will almost always // require some stack space. We grow the current (compiled) stack, then repack // the args. We finally end in a jump to the generic interpreter entry point. // On exit from the interpreter, the interpreter will restore our SP (lest the // compiled code, which relies solely on SP and not RBP, get sick).
__ bind(ok); // Method might have been compiled since the call site was patched to // interpreted if that is the case treat it as a miss so we can get // the call site corrected.
__ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
__ jcc(Assembler::equal, skip_fixup);
__ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
}
address c2i_entry = __ pc();
// Class initialization barrier for static methods
address c2i_no_clinit_check_entry = NULL; if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier; Register method = rbx;
int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
VMRegPair *regs,
VMRegPair *regs2, int total_args_passed) {
assert(regs2 == NULL, "not needed on x86"); // We return the amount of VMRegImpl stack slots we need to reserve for all // the arguments NOT counting out_preserve_stack_slots.
for (uint i = 0; i < total_args_passed; i++) {
VMReg vmreg = VEC_ArgReg[fp_args++]->as_VMReg(); int next_val = num_bits == 64 ? 1 : (num_bits == 128 ? 3 : (num_bits == 256 ? 7 : 15));
regs[i].set_pair(vmreg->next(next_val), vmreg);
}
return stk_args;
}
void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { // We always ignore the frame_slots arg and just use the space just below frame pointer // which by this time is free to use switch (ret_type) { case T_FLOAT:
__ movflt(Address(rbp, -wordSize), xmm0); break; case T_DOUBLE:
__ movdbl(Address(rbp, -wordSize), xmm0); break; case T_VOID: break; default: {
__ movptr(Address(rbp, -wordSize), rax);
}
}
}
void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { // We always ignore the frame_slots arg and just use the space just below frame pointer // which by this time is free to use switch (ret_type) { case T_FLOAT:
__ movflt(xmm0, Address(rbp, -wordSize)); break; case T_DOUBLE:
__ movdbl(xmm0, Address(rbp, -wordSize)); break; case T_VOID: break; default: {
__ movptr(rax, Address(rbp, -wordSize));
}
}
}
staticvoid save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { for ( int i = first_arg ; i < arg_count ; i++ ) { if (args[i].first()->is_Register()) {
__ push(args[i].first()->as_Register());
} elseif (args[i].first()->is_XMMRegister()) {
__ subptr(rsp, 2*wordSize);
__ movdbl(Address(rsp, 0), args[i].first()->as_XMMRegister());
}
}
}
staticvoid restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) { if (args[i].first()->is_Register()) {
__ pop(args[i].first()->as_Register());
} elseif (args[i].first()->is_XMMRegister()) {
__ movdbl(args[i].first()->as_XMMRegister(), Address(rsp, 0));
__ addptr(rsp, 2*wordSize);
}
}
}
staticvoid verify_oop_args(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, const VMRegPair* regs) { Register temp_reg = rbx; // not part of any compiled calling seq if (VerifyOops) { for (int i = 0; i < method->size_of_parameters(); i++) { if (is_reference_type(sig_bt[i])) {
VMReg r = regs[i].first();
assert(r->is_valid(), "bad oop arg"); if (r->is_stack()) {
__ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
__ verify_oop(temp_reg);
} else {
__ verify_oop(r->as_Register());
}
}
}
}
}
staticvoid check_continuation_enter_argument(VMReg actual_vmreg, Register expected_reg, constchar* name) {
assert(!actual_vmreg->is_stack(), "%s cannot be on stack", name);
assert(actual_vmreg->as_Register() == expected_reg, "%s is in unexpected register: %s instead of %s",
name, actual_vmreg->as_Register()->name(), expected_reg->name());
}
// enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) int pos_cont_obj = 0; int pos_is_cont = 1; int pos_is_virtual = 2;
// The platform-specific calling convention may present the arguments in various registers. // To simplify the rest of the code, we expect the arguments to reside at these known // registers, and we additionally check the placement here in case calling convention ever // changes. Register reg_cont_obj = c_rarg1; Register reg_is_cont = c_rarg2; Register reg_is_virtual = c_rarg3;
// i2i entry used at interp_only_mode only
interpreted_entry_offset = __ pc() - start;
{ #ifdef ASSERT
Label is_interp_only;
__ cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);
__ jcc(Assembler::notEqual, is_interp_only);
__ stop("enterSpecial interpreter entry called when not in interp_only_mode");
__ bind(is_interp_only); #endif
__ pop(rax); // return address // Read interpreter arguments into registers (this is an ad-hoc i2c adapter)
__ movptr(c_rarg1, Address(rsp, Interpreter::stackElementSize*2));
__ movl(c_rarg2, Address(rsp, Interpreter::stackElementSize*1));
__ movl(c_rarg3, Address(rsp, Interpreter::stackElementSize*0));
__ andptr(rsp, -16); // Ensure compiled code always sees stack at proper alignment
__ push(rax); // return address
__ push_cont_fastpath();
__ enter();
stack_slots = 2; // will be adjusted in setup
OopMap* map = continuation_enter_setup(masm, stack_slots); // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe, // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway.
// Make sure the call is patchable
__ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
// Emit stub for static call
CodeBuffer* cbuf = masm->code_section()->outer();
address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc()); if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
// The call needs to be resolved. There's a special case for this in // SharedRuntime::find_callee_info_helper() which calls // LinkResolver::resolve_continuation_enter() which resolves the call to // Continuation.enter(Continuation c, boolean isContinue).
__ call(resolve);
// This nop must be exactly at the PC we push into the frame info. // We use this nop for fast CodeBlob lookup, associate the OopMap // with it right away.
__ post_call_nop();
OopMap* map = new OopMap(framesize, 1);
oop_maps->add_gc_map(frame_complete, map);
// Now write the args into the outgoing interpreter space bool has_receiver = false; Register receiver_reg = noreg; int member_arg_pos = -1; Register member_reg = noreg; int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid); if (ref_kind != 0) {
member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
member_reg = rbx; // known to be free at this point
has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
} elseif (iid == vmIntrinsics::_invokeBasic) {
has_receiver = true;
} elseif (iid == vmIntrinsics::_linkToNative) {
member_arg_pos = method->size_of_parameters() - 1; // trailing NativeEntryPoint argument
member_reg = rbx; // known to be free at this point
} else {
fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
}
if (member_reg != noreg) { // Load the member_arg into register, if necessary.
SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
VMReg r = regs[member_arg_pos].first(); if (r->is_stack()) {
__ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
} else { // no data motion is needed
member_reg = r->as_Register();
}
}
if (has_receiver) { // Make sure the receiver is loaded into a register.
assert(method->size_of_parameters() > 0, "oob");
assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
VMReg r = regs[0].first();
assert(r->is_valid(), "bad receiver arg"); if (r->is_stack()) { // Porting note: This assumes that compiled calling conventions always // pass the receiver oop in a register. If this is not true on some // platform, pick a temp and load the receiver from stack.
fatal("receiver always in a register");
receiver_reg = j_rarg0; // known to be free at this point
__ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
} else { // no data motion is needed
receiver_reg = r->as_Register();
}
}
// Figure out which address we are really jumping to:
MethodHandles::generate_method_handle_dispatch(masm, iid,
receiver_reg, member_reg, /*for_compiler_entry:*/ true);
}
// --------------------------------------------------------------------------- // Generate a native wrapper for a given method. The method takes arguments // in the Java compiled code convention, marshals them to the native // convention (handlizes oops, etc), transitions to native, makes the call, // returns to java state (possibly blocking), unhandlizes any result and // returns. // // Critical native functions are a shorthand for the use of // GetPrimtiveArrayCritical and disallow the use of any other JNI // functions. The wrapper is expected to unpack the arguments before // passing them to the callee. Critical native functions leave the state _in_Java, // since they cannot stop for GC. // Some other parts of JNI setup are skipped like the tear down of the JNI handle // block and the check for pending exceptions it's impossible for them // to be thrown. //
nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, const methodHandle& method, int compile_id,
BasicType* in_sig_bt,
VMRegPair* in_regs,
BasicType ret_type) { if (method->is_continuation_native_intrinsic()) { int exception_offset = -1;
OopMapSet* oop_maps = new OopMapSet(); int frame_complete = -1; int stack_slots = -1; int interpreted_entry_offset = -1; int vep_offset = -1; if (method->is_continuation_enter_intrinsic()) {
gen_continuation_enter(masm,
in_regs,
exception_offset,
oop_maps,
frame_complete,
stack_slots,
interpreted_entry_offset,
vep_offset);
} elseif (method->is_continuation_yield_intrinsic()) {
gen_continuation_yield(masm,
in_regs,
oop_maps,
frame_complete,
stack_slots,
--> --------------------
--> maximum size reached
--> --------------------
¤ Dauer der Verarbeitung: 0.37 Sekunden
(vorverarbeitet)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung ist noch experimentell.
