| Quellcodebibliothek Statistik Leitseite products/sources/formale Sprachen/Java/Openjdk/src/hotspot/cpu/riscv/ (Sun/Oracle ©) Datei vom 13.11.2022 mit Größe 137 kB |
|
Quellcode-Bibliothek stubGenerator_riscv.cpp Sprache: C |
|||||||||
|
/*
* Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "compiler/oopMap.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" #include "interpreter/interpreter.hpp" #include "memory/universe.hpp" #include "nativeInst_riscv.hpp" #include "oops/instanceOop.hpp" #include "oops/method.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/continuation.hpp" #include "runtime/continuationEntry.inline.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaThread.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubCodeGenerator.hpp" #include "runtime/stubRoutines.hpp" #include "utilities/align.hpp" #include "utilities/powerOfTwo.hpp" #ifdef COMPILER2 #include "opto/runtime.hpp" #endif #if INCLUDE_ZGC #include "gc/z/zThreadLocalData.hpp" #endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure // see the comment in stubRoutines.hpp #undef __ #define __ _masm-> #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ #else #define BLOCK_COMMENT(str) __ block_comment(str) #endif #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") // Stub Code definitions class StubGenerator: public StubCodeGenerator { private: #ifdef PRODUCT #define inc_counter_np(counter) ((void)0) #else void inc_counter_np_(int& counter) { __ la(t1, ExternalAddress((address)&counter)); __ lwu(t0, Address(t1, 0)); __ addiw(t0, t0, 1); __ sw(t0, Address(t1, 0)); } #define inc_counter_np(counter) \ BLOCK_COMMENT("inc_counter " #counter); \ inc_counter_np_(counter); #endif // Call stubs are used to call Java from C // // Arguments: // c_rarg0: call wrapper address address // c_rarg1: result address // c_rarg2: result type BasicType // c_rarg3: method Method* // c_rarg4: (interpreter) entry point address // c_rarg5: parameters intptr_t* // c_rarg6: parameter size (in words) int // c_rarg7: thread Thread* // // There is no return from the stub itself as any Java result // is written to result // // we save x1 (ra) as the return PC at the base of the frame and // link x8 (fp) below it as the frame pointer installing sp (x2) // into fp. // // we save x10-x17, which accounts for all the c arguments. // // TODO: strictly do we need to save them all? they are treated as // volatile by C so could we omit saving the ones we are going to // place in global registers (thread? method?) or those we only use // during setup of the Java call? // // we don't need to save x5 which C uses as an indirect result location // return register. // // we don't need to save x6-x7 and x28-x31 which both C and Java treat as // volatile // // we save x9, x18-x27, f8-f9, and f18-f27 which Java uses as temporary // registers and C expects to be callee-save // // so the stub frame looks like this when we enter Java code // // [ return_from_Java ] <--- sp // [ argument word n ] // ... // -34 [ argument word 1 ] // -33 [ saved f27 ] <--- sp_after_call // -32 [ saved f26 ] // -31 [ saved f25 ] // -30 [ saved f24 ] // -29 [ saved f23 ] // -28 [ saved f22 ] // -27 [ saved f21 ] // -26 [ saved f20 ] // -25 [ saved f19 ] // -24 [ saved f18 ] // -23 [ saved f9 ] // -22 [ saved f8 ] // -21 [ saved x27 ] // -20 [ saved x26 ] // -19 [ saved x25 ] // -18 [ saved x24 ] // -17 [ saved x23 ] // -16 [ saved x22 ] // -15 [ saved x21 ] // -14 [ saved x20 ] // -13 [ saved x19 ] // -12 [ saved x18 ] // -11 [ saved x9 ] // -10 [ call wrapper (x10) ] // -9 [ result (x11) ] // -8 [ result type (x12) ] // -7 [ method (x13) ] // -6 [ entry point (x14) ] // -5 [ parameters (x15) ] // -4 [ parameter size (x16) ] // -3 [ thread (x17) ] // -2 [ saved fp (x8) ] // -1 [ saved ra (x1) ] // 0 [ ] <--- fp == saved sp (x2) // Call stub stack layout word offsets from fp enum call_stub_layout { sp_after_call_off = -33, f27_off = -33, f26_off = -32, f25_off = -31, f24_off = -30, f23_off = -29, f22_off = -28, f21_off = -27, f20_off = -26, f19_off = -25, f18_off = -24, f9_off = -23, f8_off = -22, x27_off = -21, x26_off = -20, x25_off = -19, x24_off = -18, x23_off = -17, x22_off = -16, x21_off = -15, x20_off = -14, x19_off = -13, x18_off = -12, x9_off = -11, call_wrapper_off = -10, result_off = -9, result_type_off = -8, method_off = -7, entry_point_off = -6, parameters_off = -5, parameter_size_off = -4, thread_off = -3, fp_f = -2, retaddr_off = -1, }; address generate_call_stub(address& return_address) { assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, "adjust this code"); StubCodeMark mark(this, "StubRoutines", "call_stub"); address start = __ pc(); const Address sp_after_call (fp, sp_after_call_off * wordSize); const Address call_wrapper (fp, call_wrapper_off * wordSize); const Address result (fp, result_off * wordSize); const Address result_type (fp, result_type_off * wordSize); const Address method (fp, method_off * wordSize); const Address entry_point (fp, entry_point_off * wordSize); const Address parameters (fp, parameters_off * wordSize); const Address parameter_size(fp, parameter_size_off * wordSize); const Address thread (fp, thread_off * wordSize); const Address f27_save (fp, f27_off * wordSize); const Address f26_save (fp, f26_off * wordSize); const Address f25_save (fp, f25_off * wordSize); const Address f24_save (fp, f24_off * wordSize); const Address f23_save (fp, f23_off * wordSize); const Address f22_save (fp, f22_off * wordSize); const Address f21_save (fp, f21_off * wordSize); const Address f20_save (fp, f20_off * wordSize); const Address f19_save (fp, f19_off * wordSize); const Address f18_save (fp, f18_off * wordSize); const Address f9_save (fp, f9_off * wordSize); const Address f8_save (fp, f8_off * wordSize); const Address x27_save (fp, x27_off * wordSize); const Address x26_save (fp, x26_off * wordSize); const Address x25_save (fp, x25_off * wordSize); const Address x24_save (fp, x24_off * wordSize); const Address x23_save (fp, x23_off * wordSize); const Address x22_save (fp, x22_off * wordSize); const Address x21_save (fp, x21_off * wordSize); const Address x20_save (fp, x20_off * wordSize); const Address x19_save (fp, x19_off * wordSize); const Address x18_save (fp, x18_off * wordSize); const Address x9_save (fp, x9_off * wordSize); // stub code address riscv_entry = __ pc(); // set up frame and move sp to end of save area __ enter(); __ addi(sp, fp, sp_after_call_off * wordSize); // save register parameters and Java temporary/global registers // n.b. we save thread even though it gets installed in // xthread because we want to sanity check tp later __ sd(c_rarg7, thread); __ sw(c_rarg6, parameter_size); __ sd(c_rarg5, parameters); __ sd(c_rarg4, entry_point); __ sd(c_rarg3, method); __ sd(c_rarg2, result_type); __ sd(c_rarg1, result); __ sd(c_rarg0, call_wrapper); __ sd(x9, x9_save); __ sd(x18, x18_save); __ sd(x19, x19_save); __ sd(x20, x20_save); __ sd(x21, x21_save); __ sd(x22, x22_save); __ sd(x23, x23_save); __ sd(x24, x24_save); __ sd(x25, x25_save); __ sd(x26, x26_save); __ sd(x27, x27_save); __ fsd(f8, f8_save); __ fsd(f9, f9_save); __ fsd(f18, f18_save); __ fsd(f19, f19_save); __ fsd(f20, f20_save); __ fsd(f21, f21_save); __ fsd(f22, f22_save); __ fsd(f23, f23_save); __ fsd(f24, f24_save); __ fsd(f25, f25_save); __ fsd(f26, f26_save); __ fsd(f27, f27_save); // install Java thread in global register now we have saved // whatever value it held __ mv(xthread, c_rarg7); // And method __ mv(xmethod, c_rarg3); // set up the heapbase register __ reinit_heapbase(); #ifdef ASSERT // make sure we have no pending exceptions { Label L; __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); __ beqz(t0, L); __ stop("StubRoutines::call_stub: entered with pending exception"); __ BIND(L); } #endif // pass parameters if any __ mv(esp, sp); __ slli(t0, c_rarg6, LogBytesPerWord); __ sub(t0, sp, t0); // Move SP out of the way __ andi(sp, t0, -2 * wordSize); BLOCK_COMMENT("pass parameters if any"); Label parameters_done; // parameter count is still in c_rarg6 // and parameter pointer identifying param 1 is in c_rarg5 __ beqz(c_rarg6, parameters_done); address loop = __ pc(); __ ld(t0, Address(c_rarg5, 0)); __ addi(c_rarg5, c_rarg5, wordSize); __ addi(c_rarg6, c_rarg6, -1); __ push_reg(t0); __ bgtz(c_rarg6, loop); __ BIND(parameters_done); // call Java entry -- passing methdoOop, and current sp // xmethod: Method* // x19_sender_sp: sender sp BLOCK_COMMENT("call Java function"); __ mv(x19_sender_sp, sp); __ jalr(c_rarg4); // save current address for use by exception handling code return_address = __ pc(); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) // n.b. this assumes Java returns an integral result in x10 // and a floating result in j_farg0 __ ld(j_rarg2, result); Label is_long, is_float, is_double, exit; __ ld(j_rarg1, result_type); __ mv(t0, (u1)T_OBJECT); __ beq(j_rarg1, t0, is_long); __ mv(t0, (u1)T_LONG); __ beq(j_rarg1, t0, is_long); __ mv(t0, (u1)T_FLOAT); __ beq(j_rarg1, t0, is_float); __ mv(t0, (u1)T_DOUBLE); __ beq(j_rarg1, t0, is_double); // handle T_INT case __ sw(x10, Address(j_rarg2)); __ BIND(exit); // pop parameters __ addi(esp, fp, sp_after_call_off * wordSize); #ifdef ASSERT // verify that threads correspond { Label L, S; __ ld(t0, thread); __ bne(xthread, t0, S); __ get_thread(t0); __ beq(xthread, t0, L); __ BIND(S); __ stop("StubRoutines::call_stub: threads must correspond"); __ BIND(L); } #endif __ pop_cont_fastpath(xthread); // restore callee-save registers __ fld(f27, f27_save); __ fld(f26, f26_save); __ fld(f25, f25_save); __ fld(f24, f24_save); __ fld(f23, f23_save); __ fld(f22, f22_save); __ fld(f21, f21_save); __ fld(f20, f20_save); __ fld(f19, f19_save); __ fld(f18, f18_save); __ fld(f9, f9_save); __ fld(f8, f8_save); __ ld(x27, x27_save); __ ld(x26, x26_save); __ ld(x25, x25_save); __ ld(x24, x24_save); __ ld(x23, x23_save); __ ld(x22, x22_save); __ ld(x21, x21_save); __ ld(x20, x20_save); __ ld(x19, x19_save); __ ld(x18, x18_save); __ ld(x9, x9_save); __ ld(c_rarg0, call_wrapper); __ ld(c_rarg1, result); __ ld(c_rarg2, result_type); __ ld(c_rarg3, method); __ ld(c_rarg4, entry_point); __ ld(c_rarg5, parameters); __ ld(c_rarg6, parameter_size); __ ld(c_rarg7, thread); // leave frame and return to caller __ leave(); __ ret(); // handle return types different from T_INT __ BIND(is_long); __ sd(x10, Address(j_rarg2, 0)); __ j(exit); __ BIND(is_float); __ fsw(j_farg0, Address(j_rarg2, 0), t0); __ j(exit); __ BIND(is_double); __ fsd(j_farg0, Address(j_rarg2, 0), t0); __ j(exit); return start; } // Return point for a Java call if there's an exception thrown in // Java code. The exception is caught and transformed into a // pending exception stored in JavaThread that can be tested from // within the VM. // // Note: Usually the parameters are removed by the callee. In case // of an exception crossing an activation frame boundary, that is // not the case if the callee is compiled code => need to setup the // sp. // // x10: exception oop address generate_catch_exception() { StubCodeMark mark(this, "StubRoutines", "catch_exception"); address start = __ pc(); // same as in generate_call_stub(): const Address thread(fp, thread_off * wordSize); #ifdef ASSERT // verify that threads correspond { Label L, S; __ ld(t0, thread); __ bne(xthread, t0, S); __ get_thread(t0); __ beq(xthread, t0, L); __ bind(S); __ stop("StubRoutines::catch_exception: threads must correspond"); __ bind(L); } #endif // set pending exception __ verify_oop(x10); __ sd(x10, Address(xthread, Thread::pending_exception_offset())); __ mv(t0, (address)__FILE__); __ sd(t0, Address(xthread, Thread::exception_file_offset())); __ mv(t0, (int)__LINE__); __ sw(t0, Address(xthread, Thread::exception_line_offset())); // complete return to VM assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before"); __ j(StubRoutines::_call_stub_return_address); return start; } // Continuation point for runtime calls returning with a pending // exception. The pending exception check happened in the runtime // or native call stub. The pending exception in Thread is // converted into a Java-level exception. // // Contract with Java-level exception handlers: // x10: exception // x13: throwing pc // // NOTE: At entry of this stub, exception-pc must be in RA !! // NOTE: this is always used as a jump target within generated code // so it just needs to be generated code with no x86 prolog address generate_forward_exception() { StubCodeMark mark(this, "StubRoutines", "forward exception"); address start = __ pc(); // Upon entry, RA points to the return address returning into // Java (interpreted or compiled) code; i.e., the return address // becomes the throwing pc. // // Arguments pushed before the runtime call are still on the stack // but the exception handler will reset the stack pointer -> // ignore them. A potential result in registers can be ignored as // well. #ifdef ASSERT // make sure this code is only executed if there is a pending exception { Label L; __ ld(t0, Address(xthread, Thread::pending_exception_offset())); __ bnez(t0, L); __ stop("StubRoutines::forward exception: no pending exception (1)"); __ bind(L); } #endif // compute exception handler into x9 // call the VM to find the handler address associated with the // caller address. pass thread in x10 and caller pc (ret address) // in x11. n.b. the caller pc is in ra, unlike x86 where it is on // the stack. __ mv(c_rarg1, ra); // ra will be trashed by the VM call so we move it to x9 // (callee-saved) because we also need to pass it to the handler // returned by this call. __ mv(x9, ra); BLOCK_COMMENT("call exception_handler_for_return_address"); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, c_rarg1); // we should not really care that ra is no longer the callee // address. we saved the value the handler needs in x9 so we can // just copy it to x13. however, the C2 handler will push its own // frame and then calls into the VM and the VM code asserts that // the PC for the frame above the handler belongs to a compiled // Java method. So, we restore ra here to satisfy that assert. __ mv(ra, x9); // setup x10 & x13 & clear pending exception __ mv(x13, x9); __ mv(x9, x10); __ ld(x10, Address(xthread, Thread::pending_exception_offset())); __ sd(zr, Address(xthread, Thread::pending_exception_offset())); #ifdef ASSERT // make sure exception is set { Label L; __ bnez(x10, L); __ stop("StubRoutines::forward exception: no pending exception (2)"); __ bind(L); } #endif // continue at exception handler // x10: exception // x13: throwing pc // x9: exception handler __ verify_oop(x10); __ jr(x9); return start; } // Non-destructive plausibility checks for oops // // Arguments: // x10: oop to verify // t0: error message // // Stack after saving c_rarg3: // [tos + 0]: saved c_rarg3 // [tos + 1]: saved c_rarg2 // [tos + 2]: saved ra // [tos + 3]: saved t1 // [tos + 4]: saved x10 // [tos + 5]: saved t0 address generate_verify_oop() { StubCodeMark mark(this, "StubRoutines", "verify_oop"); address start = __ pc(); Label exit, error; __ push_reg(RegSet::of(c_rarg2, c_rarg3), sp); // save c_rarg2 and c_rarg3 __ la(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr())); __ ld(c_rarg3, Address(c_rarg2)); __ add(c_rarg3, c_rarg3, 1); __ sd(c_rarg3, Address(c_rarg2)); // object is in x10 // make sure object is 'reasonable' __ beqz(x10, exit); // if obj is NULL it is OK #if INCLUDE_ZGC if (UseZGC) { // Check if mask is good. // verifies that ZAddressBadMask & x10 == 0 __ ld(c_rarg3, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); __ andr(c_rarg2, x10, c_rarg3); __ bnez(c_rarg2, error); } #endif // Check if the oop is in the right area of memory __ mv(c_rarg3, (intptr_t) Universe::verify_oop_mask()); __ andr(c_rarg2, x10, c_rarg3); __ mv(c_rarg3, (intptr_t) Universe::verify_oop_bits()); // Compare c_rarg2 and c_rarg3. __ bne(c_rarg2, c_rarg3, error); // make sure klass is 'reasonable', which is not zero. __ load_klass(x10, x10); // get klass __ beqz(x10, error); // if klass is NULL it is broken // return if everything seems ok __ bind(exit); __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 __ ret(); // handle errors __ bind(error); __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 __ push_reg(RegSet::range(x0, x31), sp); // debug(char* msg, int64_t pc, int64_t regs[]) __ mv(c_rarg0, t0); // pass address of error message __ mv(c_rarg1, ra); // pass return address __ mv(c_rarg2, sp); // pass address of regs on stack #ifndef PRODUCT assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); #endif BLOCK_COMMENT("call MacroAssembler::debug"); __ call(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)); __ ebreak(); return start; } // The inner part of zero_words(). // // Inputs: // x28: the HeapWord-aligned base address of an array to zero. // x29: the count in HeapWords, x29 > 0. // // Returns x28 and x29, adjusted for the caller to clear. // x28: the base address of the tail of words left to clear. // x29: the number of words in the tail. // x29 < MacroAssembler::zero_words_block_size. address generate_zero_blocks() { Label done; const Register base = x28, cnt = x29, tmp1 = x30, tmp2 = x31; __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", "zero_blocks"); address start = __ pc(); if (UseBlockZeroing) { // Ensure count >= 2*CacheLineSize so that it still deserves a cbo.zero // after alignment. Label small; int low_limit = MAX2(2 * CacheLineSize, BlockZeroingLowLimit) / wordSize; __ mv(tmp1, low_limit); __ blt(cnt, tmp1, small); __ zero_dcache_blocks(base, cnt, tmp1, tmp2); __ bind(small); } { // Clear the remaining blocks. Label loop; __ mv(tmp1, MacroAssembler::zero_words_block_size); __ blt(cnt, tmp1, done); __ bind(loop); for (int i = 0; i < MacroAssembler::zero_words_block_size; i++) { __ sd(zr, Address(base, i * wordSize)); } __ add(base, base, MacroAssembler::zero_words_block_size * wordSize); __ sub(cnt, cnt, MacroAssembler::zero_words_block_size); __ bge(cnt, tmp1, loop); __ bind(done); } __ ret(); return start; } typedef enum { copy_forwards = 1, copy_backwards = -1 } copy_direction; // Bulk copy of blocks of 8 words. // // count is a count of words. // // Precondition: count >= 8 // // Postconditions: // // The least significant bit of count contains the remaining count // of words to copy. The rest of count is trash. // // s and d are adjusted to point to the remaining words to copy // void generate_copy_longs(Label &start, Register s, Register d, Register count, copy_direction direction) { int unit = wordSize * direction; int bias = wordSize; const Register tmp_reg0 = x13, tmp_reg1 = x14, tmp_reg2 = x15, tmp_reg3 = x16, tmp_reg4 = x17, tmp_reg5 = x7, tmp_reg6 = x28, tmp_reg7 = x29; const Register stride = x30; assert_different_registers(t0, tmp_reg0, tmp_reg1, tmp_reg2, tmp_reg3, tmp_reg4, tmp_reg5, tmp_reg6, tmp_reg7); assert_different_registers(s, d, count, t0); Label again, drain; const char* stub_name = NULL; if (direction == copy_forwards) { stub_name = "forward_copy_longs"; } else { stub_name = "backward_copy_longs"; } StubCodeMark mark(this, "StubRoutines", stub_name); __ align(CodeEntryAlignment); __ bind(start); if (direction == copy_forwards) { __ sub(s, s, bias); __ sub(d, d, bias); } #ifdef ASSERT // Make sure we are never given < 8 words { Label L; __ mv(t0, 8); __ bge(count, t0, L); __ stop("genrate_copy_longs called with < 8 words"); __ bind(L); } #endif __ ld(tmp_reg0, Address(s, 1 * unit)); __ ld(tmp_reg1, Address(s, 2 * unit)); __ ld(tmp_reg2, Address(s, 3 * unit)); __ ld(tmp_reg3, Address(s, 4 * unit)); __ ld(tmp_reg4, Address(s, 5 * unit)); __ ld(tmp_reg5, Address(s, 6 * unit)); __ ld(tmp_reg6, Address(s, 7 * unit)); __ ld(tmp_reg7, Address(s, 8 * unit)); __ addi(s, s, 8 * unit); __ sub(count, count, 16); __ bltz(count, drain); __ bind(again); __ sd(tmp_reg0, Address(d, 1 * unit)); __ sd(tmp_reg1, Address(d, 2 * unit)); __ sd(tmp_reg2, Address(d, 3 * unit)); __ sd(tmp_reg3, Address(d, 4 * unit)); __ sd(tmp_reg4, Address(d, 5 * unit)); __ sd(tmp_reg5, Address(d, 6 * unit)); __ sd(tmp_reg6, Address(d, 7 * unit)); __ sd(tmp_reg7, Address(d, 8 * unit)); __ ld(tmp_reg0, Address(s, 1 * unit)); __ ld(tmp_reg1, Address(s, 2 * unit)); __ ld(tmp_reg2, Address(s, 3 * unit)); __ ld(tmp_reg3, Address(s, 4 * unit)); __ ld(tmp_reg4, Address(s, 5 * unit)); __ ld(tmp_reg5, Address(s, 6 * unit)); __ ld(tmp_reg6, Address(s, 7 * unit)); __ ld(tmp_reg7, Address(s, 8 * unit)); __ addi(s, s, 8 * unit); __ addi(d, d, 8 * unit); __ sub(count, count, 8); __ bgez(count, again); // Drain __ bind(drain); __ sd(tmp_reg0, Address(d, 1 * unit)); __ sd(tmp_reg1, Address(d, 2 * unit)); __ sd(tmp_reg2, Address(d, 3 * unit)); __ sd(tmp_reg3, Address(d, 4 * unit)); __ sd(tmp_reg4, Address(d, 5 * unit)); __ sd(tmp_reg5, Address(d, 6 * unit)); __ sd(tmp_reg6, Address(d, 7 * unit)); __ sd(tmp_reg7, Address(d, 8 * unit)); __ addi(d, d, 8 * unit); { Label L1, L2; __ andi(t0, count, 4); __ beqz(t0, L1); __ ld(tmp_reg0, Address(s, 1 * unit)); __ ld(tmp_reg1, Address(s, 2 * unit)); __ ld(tmp_reg2, Address(s, 3 * unit)); __ ld(tmp_reg3, Address(s, 4 * unit)); __ addi(s, s, 4 * unit); __ sd(tmp_reg0, Address(d, 1 * unit)); __ sd(tmp_reg1, Address(d, 2 * unit)); __ sd(tmp_reg2, Address(d, 3 * unit)); __ sd(tmp_reg3, Address(d, 4 * unit)); __ addi(d, d, 4 * unit); __ bind(L1); if (direction == copy_forwards) { __ addi(s, s, bias); __ addi(d, d, bias); } __ andi(t0, count, 2); __ beqz(t0, L2); if (direction == copy_backwards) { __ addi(s, s, 2 * unit); __ ld(tmp_reg0, Address(s)); __ ld(tmp_reg1, Address(s, wordSize)); __ addi(d, d, 2 * unit); __ sd(tmp_reg0, Address(d)); __ sd(tmp_reg1, Address(d, wordSize)); } else { __ ld(tmp_reg0, Address(s)); __ ld(tmp_reg1, Address(s, wordSize)); __ addi(s, s, 2 * unit); __ sd(tmp_reg0, Address(d)); __ sd(tmp_reg1, Address(d, wordSize)); __ addi(d, d, 2 * unit); } __ bind(L2); } __ ret(); } Label copy_f, copy_b; // All-singing all-dancing memory copy. // // Copy count units of memory from s to d. The size of a unit is // step, which can be positive or negative depending on the direction // of copy. If is_aligned is false, we align the source address. // /* * if (is_aligned) { * if (count >= 32) * goto copy32_loop; * if (count >= 8) * goto copy8_loop; * goto copy_small; * } * bool is_backwards = step < 0; * int granularity = uabs(step); * count = count * granularity; * count bytes * * if (is_backwards) { * s += count; * d += count; * } * * count limit maybe greater than 16, for better performance * if (count < 16) { * goto copy_small; * } * * if ((dst % 8) == (src % 8)) { * aligned; * goto copy_big; * } * * copy_big: * if the amount to copy is more than (or equal to) 32 bytes goto copy32_loop * else goto copy8_loop * copy_small: * load element one by one; * done; */ typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp void copy_memory_v(Register s, Register d, Register count, Register tmp, int step) { bool is_backward = step < 0; int granularity = uabs(step); const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17; assert_different_registers(s, d, cnt, vl, tmp, tmp1, tmp2); Assembler::SEW sew = Assembler::elembytes_to_sew(granularity); Label loop_forward, loop_backward, done; __ mv(dst, d); __ mv(src, s); __ mv(cnt, count); __ bind(loop_forward); __ vsetvli(vl, cnt, sew, Assembler::m8); if (is_backward) { __ bne(vl, cnt, loop_backward); } __ vlex_v(v0, src, sew); __ sub(cnt, cnt, vl); __ slli(vl, vl, (int)sew); __ add(src, src, vl); __ vsex_v(v0, dst, sew); __ add(dst, dst, vl); __ bnez(cnt, loop_forward); if (is_backward) { __ j(done); __ bind(loop_backward); __ sub(tmp, cnt, vl); __ slli(tmp, tmp, sew); __ add(tmp1, s, tmp); __ vlex_v(v0, tmp1, sew); __ add(tmp2, d, tmp); __ vsex_v(v0, tmp2, sew); __ sub(cnt, cnt, vl); __ bnez(cnt, loop_forward); __ bind(done); } } void copy_memory(bool is_aligned, Register s, Register d, Register count, Register tmp, int step) { if (UseRVV) { return copy_memory_v(s, d, count, tmp, step); } bool is_backwards = step < 0; int granularity = uabs(step); const Register src = x30, dst = x31, cnt = x15, tmp3 = x16, tmp4 = x17, tmp5 = x14, tmp6 = x13; Label same_aligned; Label copy_big, copy32_loop, copy8_loop, copy_small, done; copy_insn ld_arr = NULL, st_arr = NULL; switch (granularity) { case 1 : ld_arr = (copy_insn)&MacroAssembler::lbu; st_arr = (copy_insn)&MacroAssembler::sb; break; case 2 : ld_arr = (copy_insn)&MacroAssembler::lhu; st_arr = (copy_insn)&MacroAssembler::sh; break; case 4 : ld_arr = (copy_insn)&MacroAssembler::lwu; st_arr = (copy_insn)&MacroAssembler::sw; break; case 8 : ld_arr = (copy_insn)&MacroAssembler::ld; st_arr = (copy_insn)&MacroAssembler::sd; break; default : ShouldNotReachHere(); } __ beqz(count, done); __ slli(cnt, count, exact_log2(granularity)); if (is_backwards) { __ add(src, s, cnt); __ add(dst, d, cnt); } else { __ mv(src, s); __ mv(dst, d); } if (is_aligned) { __ addi(tmp, cnt, -32); __ bgez(tmp, copy32_loop); __ addi(tmp, cnt, -8); __ bgez(tmp, copy8_loop); __ j(copy_small); } else { __ mv(tmp, 16); __ blt(cnt, tmp, copy_small); __ xorr(tmp, src, dst); __ andi(tmp, tmp, 0b111); __ bnez(tmp, copy_small); __ bind(same_aligned); __ andi(tmp, src, 0b111); __ beqz(tmp, copy_big); if (is_backwards) { __ addi(src, src, step); __ addi(dst, dst, step); } (_masm->*ld_arr)(tmp3, Address(src), t0); (_masm->*st_arr)(tmp3, Address(dst), t0); if (!is_backwards) { __ addi(src, src, step); __ addi(dst, dst, step); } __ addi(cnt, cnt, -granularity); __ beqz(cnt, done); __ j(same_aligned); __ bind(copy_big); __ mv(tmp, 32); __ blt(cnt, tmp, copy8_loop); } __ bind(copy32_loop); if (is_backwards) { __ addi(src, src, -wordSize * 4); __ addi(dst, dst, -wordSize * 4); } // we first load 32 bytes, then write it, so the direction here doesn't matter __ ld(tmp3, Address(src)); __ ld(tmp4, Address(src, 8)); __ ld(tmp5, Address(src, 16)); __ ld(tmp6, Address(src, 24)); __ sd(tmp3, Address(dst)); __ sd(tmp4, Address(dst, 8)); __ sd(tmp5, Address(dst, 16)); __ sd(tmp6, Address(dst, 24)); if (!is_backwards) { __ addi(src, src, wordSize * 4); __ addi(dst, dst, wordSize * 4); } __ addi(tmp, cnt, -(32 + wordSize * 4)); __ addi(cnt, cnt, -wordSize * 4); __ bgez(tmp, copy32_loop); // cnt >= 32, do next loop __ beqz(cnt, done); // if that's all - done __ addi(tmp, cnt, -8); // if not - copy the reminder __ bltz(tmp, copy_small); // cnt < 8, go to copy_small, else fall throught to copy8_loop __ bind(copy8_loop); if (is_backwards) { __ addi(src, src, -wordSize); __ addi(dst, dst, -wordSize); } __ ld(tmp3, Address(src)); __ sd(tmp3, Address(dst)); if (!is_backwards) { __ addi(src, src, wordSize); __ addi(dst, dst, wordSize); } __ addi(tmp, cnt, -(8 + wordSize)); __ addi(cnt, cnt, -wordSize); __ bgez(tmp, copy8_loop); // cnt >= 8, do next loop __ beqz(cnt, done); // if that's all - done __ bind(copy_small); if (is_backwards) { __ addi(src, src, step); __ addi(dst, dst, step); } (_masm->*ld_arr)(tmp3, Address(src), t0); (_masm->*st_arr)(tmp3, Address(dst), t0); if (!is_backwards) { __ addi(src, src, step); __ addi(dst, dst, step); } __ addi(cnt, cnt, -granularity); __ bgtz(cnt, copy_small); __ bind(done); } // Scan over array at a for count oops, verifying each one. // Preserves a and count, clobbers t0 and t1. void verify_oop_array(size_t size, Register a, Register count, Register temp) { Label loop, end; __ mv(t1, zr); __ slli(t0, count, exact_log2(size)); __ bind(loop); __ bgeu(t1, t0, end); __ add(temp, a, t1); if (size == (size_t)wordSize) { __ ld(temp, Address(temp, 0)); __ verify_oop(temp); } else { __ lwu(temp, Address(temp, 0)); __ decode_heap_oop(temp); // calls verify_oop } __ add(t1, t1, size); __ j(loop); __ bind(end); } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // is_oop - true => oop array, so generate store check code // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let // the hardware handle it. The two dwords within qwords that span // cache line boundaries will still be loaded and stored atomically. // // Side Effects: // disjoint_int_copy_entry is set to the no-overlap entry point // used by generate_conjoint_int_oop_copy(). // address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address* entry, const char* name, bool dest_uninitialized = false) { const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; RegSet saved_reg = RegSet::of(s, d, count); __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); __ enter(); if (entry != NULL) { *entry = __ pc(); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } if (aligned) { decorators |= ARRAYCOPY_ALIGNED; } BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg); if (is_oop) { // save regs before copy_memory __ push_reg(RegSet::of(d, count), sp); } { // UnsafeCopyMemory page error: continue after ucm bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); UnsafeCopyMemoryMark ucmm(this, add_entry, true); copy_memory(aligned, s, d, count, t0, size); } if (is_oop) { __ pop_reg(RegSet::of(d, count), sp); if (VerifyOops) { verify_oop_array(size, d, count, t2); } } bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); __ leave(); __ mv(x10, zr); // return 0 __ ret(); return start; } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // is_oop - true => oop array, so generate store check code // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let // the hardware handle it. The two dwords within qwords that span // cache line boundaries will still be loaded and stored atomically. // address generate_conjoint_copy(size_t size, bool aligned, bool is_oop, address nooverlap address* entry, const char* name, bool dest_uninitialized = false) { const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; RegSet saved_regs = RegSet::of(s, d, count); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); __ enter(); if (entry != NULL) { *entry = __ pc(); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } // use fwd copy when (d-s) above_equal (count*size) __ sub(t0, d, s); __ slli(t1, count, exact_log2(size)); __ bgeu(t0, t1, nooverlap_target); DecoratorSet decorators = IN_HEAP | IS_ARRAY; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } if (aligned) { decorators |= ARRAYCOPY_ALIGNED; } BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs); if (is_oop) { // save regs before copy_memory __ push_reg(RegSet::of(d, count), sp); } { // UnsafeCopyMemory page error: continue after ucm bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); UnsafeCopyMemoryMark ucmm(this, add_entry, true); copy_memory(aligned, s, d, count, t0, -size); } if (is_oop) { __ pop_reg(RegSet::of(d, count), sp); if (VerifyOops) { verify_oop_array(size, d, count, t2); } } bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); __ leave(); __ mv(x10, zr); // return 0 __ ret(); return start; } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, // we let the hardware handle it. The one to eight bytes within words, // dwords or qwords that span cache line boundaries will still be loaded // and stored atomically. // // Side Effects: // disjoint_byte_copy_entry is set to the no-overlap entry point // // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, // we let the hardware handle it. The one to eight bytes within words, // dwords or qwords that span cache line boundaries will still be loaded // and stored atomically. // // Side Effects: // disjoint_byte_copy_entry is set to the no-overlap entry point // used by generate_conjoint_byte_copy(). // address generate_disjoint_byte_copy(bool aligned, address* entry, const char* name) { const bool not_oop = false; return generate_disjoint_copy(sizeof (jbyte), aligned, not_oop, entry, name); } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, // we let the hardware handle it. The one to eight bytes within words, // dwords or qwords that span cache line boundaries will still be loaded // and stored atomically. // address generate_conjoint_byte_copy(bool aligned, address nooverlap_target, address* entry, const char* name) { const bool not_oop = false; return generate_conjoint_copy(sizeof (jbyte), aligned, not_oop, nooverlap_target, entr } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we // let the hardware handle it. The two or four words within dwords // or qwords that span cache line boundaries will still be loaded // and stored atomically. // // Side Effects: // disjoint_short_copy_entry is set to the no-overlap entry point // used by generate_conjoint_short_copy(). // address generate_disjoint_short_copy(bool aligned, address* entry, const char* name) { const bool not_oop = false; return generate_disjoint_copy(sizeof (jshort), aligned, not_oop, entry, name); } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we // let the hardware handle it. The two or four words within dwords // or qwords that span cache line boundaries will still be loaded // and stored atomically. // address generate_conjoint_short_copy(bool aligned, address nooverlap_target, address* entry, const char* name) { const bool not_oop = false; return generate_conjoint_copy(sizeof (jshort), aligned, not_oop, nooverlap_target, ent } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let // the hardware handle it. The two dwords within qwords that span // cache line boundaries will still be loaded and stored atomically. // // Side Effects: // disjoint_int_copy_entry is set to the no-overlap entry point // used by generate_conjoint_int_oop_copy(). // address generate_disjoint_int_copy(bool aligned, address* entry, const char* name, bool dest_uninitialized = false) { const bool not_oop = false; return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name); } // Arguments: // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let // the hardware handle it. The two dwords within qwords that span // cache line boundaries will still be loaded and stored atomically. // address generate_conjoint_int_copy(bool aligned, address nooverlap_target, address* entry, const char* name, bool dest_uninitialized = false) { const bool not_oop = false; return generate_conjoint_copy(sizeof (jint), aligned, not_oop, nooverlap_target, entry } // Arguments: // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as size_t, can be zero // // Side Effects: // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the // no-overlap entry point used by generate_conjoint_long_oop_copy(). // address generate_disjoint_long_copy(bool aligned, address* entry, const char* name, bool dest_uninitialized = false) { const bool not_oop = false; return generate_disjoint_copy(sizeof (jlong), aligned, not_oop, entry, name); } // Arguments: // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as size_t, can be zero // address generate_conjoint_long_copy(bool aligned, address nooverlap_target, address* entry, const char* name, bool dest_uninitialized = false) { const bool not_oop = false; return generate_conjoint_copy(sizeof (jlong), aligned, not_oop, nooverlap_target, entr } // Arguments: // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as size_t, can be zero // // Side Effects: // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the // no-overlap entry point used by generate_conjoint_long_oop_copy(). // address generate_disjoint_oop_copy(bool aligned, address* entry, const char* name, bool dest_uninitialized) { const bool is_oop = true; const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); return generate_disjoint_copy(size, aligned, is_oop, entry, name, dest_uninitialized); } // Arguments: // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes // ignored // name - stub name string // // Inputs: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as size_t, can be zero // address generate_conjoint_oop_copy(bool aligned, address nooverlap_target, address* entry, const char* name, bool dest_uninitialized) { const bool is_oop = true; const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); return generate_conjoint_copy(size, aligned, is_oop, nooverlap_target, entry, name, dest_uninitialized); } // Helper for generating a dynamic type check. // Smashes t0, t1. void generate_type_check(Register sub_klass, Register super_check_offset, Register super_klass, Label& L_success) { assert_different_registers(sub_klass, super_check_offset, super_klass); BLOCK_COMMENT("type_check:"); Label L_miss; __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL, super __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL); // Fall through on failure! __ BIND(L_miss); } // // Generate checkcasting array copy stub // // Input: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // c_rarg3 - size_t ckoff (super_check_offset) // c_rarg4 - oop ckval (super_klass) // // Output: // x10 == 0 - success // x10 == -1^K - failure, where K is partial transfer count // address generate_checkcast_copy(const char* name, address* entry, bool dest_uninitialized = false) { Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; // Input registers (after setup_arg_regs) const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address const Register count = c_rarg2; // elementscount const Register ckoff = c_rarg3; // super_check_offset const Register ckval = c_rarg4; // super_klass RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4); RegSet wb_post_saved_regs = RegSet::of(count); // Registers used as temps (x7, x9, x18 are save-on-entry) const Register count_save = x19; // orig elementscount const Register start_to = x18; // destination array start address const Register copied_oop = x7; // actual oop copied const Register r9_klass = x9; // oop._klass //--------------------------------------------------------------- // Assembler stub will be used for this call to arraycopy // if the two arrays are subtypes of Object[] but the // destination array type is not equal to or a supertype // of the source type. Each element must be separately // checked. assert_different_registers(from, to, count, ckoff, ckval, start_to, copied_oop, r9_klass, count_save); __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame // Caller of this entry point must set up the argument registers. if (entry != NULL) { *entry = __ pc(); BLOCK_COMMENT("Entry:"); } // Empty array: Nothing to do __ beqz(count, L_done); __ push_reg(RegSet::of(x7, x9, x18, x19), sp); #ifdef ASSERT BLOCK_COMMENT("assert consistent ckoff/ckval"); // The ckoff and ckval must be mutually consistent, // even though caller generates both. { Label L; int sco_offset = in_bytes(Klass::super_check_offset_offset()); __ lwu(start_to, Address(ckval, sco_offset)); __ beq(ckoff, start_to, L); __ stop("super_check_offset inconsistent"); __ bind(L); } #endif //ASSERT DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT; bool is_oop = true; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs); // save the original count __ mv(count_save, count); // Copy from low to high addresses __ mv(start_to, to); // Save destination array start address __ j(L_load_element); // ======== begin loop ======== // (Loop is rotated; its entry is L_load_element.) // Loop control: // for count to 0 do // copied_oop = load_heap_oop(from++) // ... generate_type_check ... // store_heap_oop(to++, copied_oop) // end __ align(OptoLoopAlignment); __ BIND(L_store_element); __ store_heap_oop(Address(to, 0), copied_oop, noreg, noreg, noreg, AS_RAW); // store the oo __ add(to, to, UseCompressedOops ? 4 : 8); __ sub(count, count, 1); __ beqz(count, L_do_card_marks); // ======== loop entry is here ======== __ BIND(L_load_element); __ load_heap_oop(copied_oop, Address(from, 0), noreg, noreg, AS_RAW); // load the oop __ add(from, from, UseCompressedOops ? 4 : 8); __ beqz(copied_oop, L_store_element); __ load_klass(r9_klass, copied_oop);// query the object klass generate_type_check(r9_klass, ckoff, ckval, L_store_element); // ======== end loop ======== // It was a real error; we must depend on the caller to finish the job. // Register count = remaining oops, count_orig = total oops. // Emit GC store barriers for the oops we have copied and report // their number to the caller. __ sub(count, count_save, count); // K = partially copied oop count __ xori(count, count, -1); // report (-1^K) to caller __ beqz(count, L_done_pop); __ BIND(L_do_card_marks); bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, t0, wb_post_sav __ bind(L_done_pop); __ pop_reg(RegSet::of(x7, x9, x18, x19), sp); inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); __ bind(L_done); __ mv(x10, count); __ leave(); __ ret(); return start; } // Perform range checks on the proposed arraycopy. // Kills temp, but nothing else. // Also, clean the sign bits of src_pos and dst_pos. void arraycopy_range_checks(Register src, // source array oop (c_rarg0) Register src_pos, // source position (c_rarg1) Register dst, // destination array oo (c_rarg2) Register dst_pos, // destination position (c_rarg3) Register length, Register temp, Label& L_failed) { BLOCK_COMMENT("arraycopy_range_checks:"); assert_different_registers(t0, temp); // if [src_pos + length > arrayOop(src)->length()] then FAIL __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes())); __ addw(temp, length, src_pos); __ bgtu(temp, t0, L_failed); // if [dst_pos + length > arrayOop(dst)->length()] then FAIL __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes())); __ addw(temp, length, dst_pos); __ bgtu(temp, t0, L_failed); // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'. __ zero_extend(src_pos, src_pos, 32); __ zero_extend(dst_pos, dst_pos, 32); BLOCK_COMMENT("arraycopy_range_checks done"); } // // Generate 'unsafe' array copy stub // Though just as safe as the other stubs, it takes an unscaled // size_t argument instead of an element count. // // Input: // c_rarg0 - source array address // c_rarg1 - destination array address // c_rarg2 - byte count, treated as ssize_t, can be zero // // Examines the alignment of the operands and dispatches // to a long, int, short, or byte copy loop. // address generate_unsafe_copy(const char* name, address byte_copy_entry, address short_copy_entry, address int_copy_entry, address long_copy_entry) { assert_cond(byte_copy_entry != NULL && short_copy_entry != NULL && int_copy_entry != NULL && long_copy_entry != NULL); Label L_long_aligned, L_int_aligned, L_short_aligned; const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); __ orr(t0, s, d); __ orr(t0, t0, count); __ andi(t0, t0, BytesPerLong - 1); __ beqz(t0, L_long_aligned); __ andi(t0, t0, BytesPerInt - 1); __ beqz(t0, L_int_aligned); __ andi(t0, t0, 1); __ beqz(t0, L_short_aligned); __ j(RuntimeAddress(byte_copy_entry)); __ BIND(L_short_aligned); __ srli(count, count, LogBytesPerShort); // size => short_count __ j(RuntimeAddress(short_copy_entry)); __ BIND(L_int_aligned); __ srli(count, count, LogBytesPerInt); // size => int_count __ j(RuntimeAddress(int_copy_entry)); __ BIND(L_long_aligned); __ srli(count, count, LogBytesPerLong); // size => long_count __ j(RuntimeAddress(long_copy_entry)); return start; } // // Generate generic array copy stubs // // Input: // c_rarg0 - src oop // c_rarg1 - src_pos (32-bits) // c_rarg2 - dst oop // c_rarg3 - dst_pos (32-bits) // c_rarg4 - element count (32-bits) // // Output: // x10 == 0 - success // x10 == -1^K - failure, where K is partial transfer count // address generate_generic_copy(const char* name, address byte_copy_entry, address short_copy_entry, address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { assert_cond(byte_copy_entry != NULL && short_copy_entry != NULL && int_copy_entry != NULL && oop_copy_entry != NULL && long_copy_entry != NULL && checkcast_copy_entry != NULL); Label L_failed, L_failed_0, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; // Input registers const Register src = c_rarg0; // source array oop const Register src_pos = c_rarg1; // source position const Register dst = c_rarg2; // destination array oop const Register dst_pos = c_rarg3; // destination position const Register length = c_rarg4; // Registers used as temps const Register dst_klass = c_rarg5; __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_generic_array_copy_ctr); //----------------------------------------------------------------------- // Assembler stub will be used for this call to arraycopy // if the following conditions are met: // // (1) src and dst must not be null. // (2) src_pos must not be negative. // (3) dst_pos must not be negative. // (4) length must not be negative. // (5) src klass and dst klass should be the same and not NULL. // (6) src and dst should be arrays. // (7) src_pos + length must not exceed length of src. // (8) dst_pos + length must not exceed length of dst. // // if [src == NULL] then return -1 __ beqz(src, L_failed); // if [src_pos < 0] then return -1 // i.e. sign bit set __ andi(t0, src_pos, 1UL << 31); __ bnez(t0, L_failed); // if [dst == NULL] then return -1 __ beqz(dst, L_failed); // if [dst_pos < 0] then return -1 // i.e. sign bit set __ andi(t0, dst_pos, 1UL << 31); __ bnez(t0, L_failed); // registers used as temp const Register scratch_length = x28; // elements count to copy const Register scratch_src_klass = x29; // array klass const Register lh = x30; // layout helper // if [length < 0] then return -1 __ addw(scratch_length, length, zr); // length (elements count, 32-bits value) // i.e. sign bit set __ andi(t0, scratch_length, 1UL << 31); __ bnez(t0, L_failed); __ load_klass(scratch_src_klass, src); #ifdef ASSERT { BLOCK_COMMENT("assert klasses not null {"); Label L1, L2; __ bnez(scratch_src_klass, L2); // it is broken if klass is NULL __ bind(L1); __ stop("broken null klass"); __ bind(L2); __ load_klass(t0, dst, t1); __ beqz(t0, L1); // this would be broken also BLOCK_COMMENT("} assert klasses not null done"); } #endif // Load layout helper (32-bits) // // |array_tag| | header_size | element_type | |log2_element_size| // 32 30 24 16 8 2 0 // // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 // const int lh_offset = in_bytes(Klass::layout_helper_offset()); // Handle objArrays completely differently... const jint objArray_lh = Klass::array_layout_helper(T_OBJECT); __ lw(lh, Address(scratch_src_klass, lh_offset)); __ mvw(t0, objArray_lh); __ beq(lh, t0, L_objArray); // if [src->klass() != dst->klass()] then return -1 __ load_klass(t1, dst); __ bne(t1, scratch_src_klass, L_failed); // if [src->is_Array() != NULL] then return -1 // i.e. (lh >= 0) __ andi(t0, lh, 1UL << 31); __ beqz(t0, L_failed); // At this point, it is known to be a typeArray (array_tag 0x3). #ifdef ASSERT { BLOCK_COMMENT("assert primitive array {"); Label L; __ mvw(t1, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift); __ bge(lh, t1, L); __ stop("must be a primitive array"); __ bind(L); BLOCK_COMMENT("} assert primitive array done"); } #endif arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, t1, L_failed); // TypeArrayKlass // // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize) // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize) // const Register t0_offset = t0; // array offset const Register x22_elsize = lh; // element size // Get array_header_in_bytes() int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1); int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width; __ slli(t0_offset, lh, XLEN - lh_header_size_msb); // left shift to remove 24 ~ 32; __ srli(t0_offset, t0_offset, XLEN - lh_header_size_width); // array_offset __ add(src, src, t0_offset); // src array offset __ add(dst, dst, t0_offset); // dst array offset BLOCK_COMMENT("choose copy loop based on element size"); // next registers should be set before the jump to corresponding stub const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address const Register count = c_rarg2; // elements count // 'from', 'to', 'count' registers should be set in such order // since they are the same as 'src', 'src_pos', 'dst'. assert(Klass::_lh_log2_element_size_shift == 0, "fix this code"); // The possible values of elsize are 0-3, i.e. exact_log2(element // size in bytes). We do a simple bitwise binary search. __ BIND(L_copy_bytes); __ andi(t0, x22_elsize, 2); __ bnez(t0, L_copy_ints); __ andi(t0, x22_elsize, 1); __ bnez(t0, L_copy_shorts); __ add(from, src, src_pos); // src_addr __ add(to, dst, dst_pos); // dst_addr __ addw(count, scratch_length, zr); // length __ j(RuntimeAddress(byte_copy_entry)); __ BIND(L_copy_shorts); __ shadd(from, src_pos, src, t0, 1); // src_addr __ shadd(to, dst_pos, dst, t0, 1); // dst_addr __ addw(count, scratch_length, zr); // length __ j(RuntimeAddress(short_copy_entry)); __ BIND(L_copy_ints); __ andi(t0, x22_elsize, 1); __ bnez(t0, L_copy_longs); __ shadd(from, src_pos, src, t0, 2); // src_addr __ shadd(to, dst_pos, dst, t0, 2); // dst_addr __ addw(count, scratch_length, zr); // length __ j(RuntimeAddress(int_copy_entry)); __ BIND(L_copy_longs); #ifdef ASSERT { BLOCK_COMMENT("assert long copy {"); Label L; __ andi(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> x22_elsize __ addw(lh, lh, zr); __ mvw(t0, LogBytesPerLong); __ beq(x22_elsize, t0, L); __ stop("must be long copy, but elsize is wrong"); __ bind(L); BLOCK_COMMENT("} assert long copy done"); } #endif __ shadd(from, src_pos, src, t0, 3); // src_addr __ shadd(to, dst_pos, dst, t0, 3); // dst_addr __ addw(count, scratch_length, zr); // length __ j(RuntimeAddress(long_copy_entry)); // ObjArrayKlass __ BIND(L_objArray); --> -------------------- --> maximum size reached --> -------------------- ¤ Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.0.33Bemerkung: (vorverarbeitet) ¤*Bot Zugriff |
|
2026-03-28