__ Bind(GetEntryLabel()); // The `src_curr_addr` and `dst_curr_addr` were initialized before entering the slow-path.
GenArrayAddress(assembler, src_stop_addr, src_curr_addr, length, type, /*data_offset=*/ 0u);
NearLabel loop;
__ Bind(&loop);
__ movl(CpuRegister(TMP), Address(src_curr_addr, 0));
__ MaybeUnpoisonHeapReference(CpuRegister(TMP)); // TODO: Inline the mark bit check before calling the runtime? // TMP = ReadBarrier::Mark(TMP); // No need to save live registers; it's taken care of by the // entrypoint. Also, there is no need to update the stack mask, // as this runtime call will not trigger a garbage collection.
int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(TMP); // This runtime call does not require a stack map.
x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
__ MaybePoisonHeapReference(CpuRegister(TMP));
__ movl(Address(dst_curr_addr, 0), CpuRegister(TMP));
__ addl(src_curr_addr, Immediate(element_size));
__ addl(dst_curr_addr, Immediate(element_size));
__ cmpl(src_curr_addr, src_stop_addr);
__ j(kNotEqual, &loop);
__ jmp(GetExitLabel());
}
// The MethodHandle.invokeExact intrinsic sets up arguments to match the target method call. If we // need to go to the slow path, we call art_quick_invoke_polymorphic_with_hidden_receiver, which // expects the MethodHandle object in RDI (in place of the actual ArtMethod). class InvokePolymorphicSlowPathX86_64 : public SlowPathCode { public:
InvokePolymorphicSlowPathX86_64(HInstruction* instruction, CpuRegister method_handle)
: SlowPathCode(instruction), method_handle_(method_handle) {
DCHECK(instruction->IsInvokePolymorphic());
}
void IntrinsicCodeGeneratorX86_64::VisitMathSqrt(HInvoke* invoke) {
LocationSummary* locations = invoke->GetLocations();
XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>();
XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
GetAssembler()->sqrtsd(out, in);
}
staticvoid CreateSSE41FPToFPLocations(ArenaAllocator* allocator,
HInvoke* invoke, const CodeGeneratorX86_64* codegen) { // Do we have instruction support? if (!codegen->GetInstructionSetFeatures().HasSSE4_1()) { return;
}
CreateFPToFPLocations(allocator, invoke);
}
staticvoid GenSSE41FPToFPIntrinsic(HInvoke* invoke, X86_64Assembler* assembler, int round_mode) {
LocationSummary* locations = invoke->GetLocations();
DCHECK(!locations->WillCall());
XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>();
XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
__ roundsd(out, in, Immediate(round_mode));
}
staticvoid CreateSSE41FPToIntLocations(ArenaAllocator* allocator,
HInvoke* invoke, const CodeGeneratorX86_64* codegen) { // Do we have instruction support? if (!codegen->GetInstructionSetFeatures().HasSSE4_1()) { return;
}
// Since no direct x86 rounding instruction matches the required semantics, // this intrinsic is implemented as follows: // result = floor(in); // if (in - result >= 0.5f) // result = result + 1.0f;
__ movss(t2, in);
__ roundss(t1, in, Immediate(1));
__ subss(t2, t1);
__ comiss(t2, codegen_->LiteralFloatAddress(0.5f));
__ j(kBelow, &skip_incr);
__ addss(t1, codegen_->LiteralFloatAddress(1.0f));
__ Bind(&skip_incr);
// Final conversion to an integer. Unfortunately this also does not have a // direct x86 instruction, since NaN should map to 0 and large positive // values need to be clipped to the extreme value.
codegen_->Load32BitValue(out, kPrimIntMax);
__ cvtsi2ss(t2, out);
__ comiss(t1, t2);
__ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered
__ movl(out, Immediate(0)); // does not change flags
__ j(kUnordered, &done); // NaN mapped to 0 (just moved in out)
__ cvttss2si(out, t1);
__ Bind(&done);
}
// Since no direct x86 rounding instruction matches the required semantics, // this intrinsic is implemented as follows: // result = floor(in); // if (in - result >= 0.5) // result = result + 1.0f;
__ movsd(t2, in);
__ roundsd(t1, in, Immediate(1));
__ subsd(t2, t1);
__ comisd(t2, codegen_->LiteralDoubleAddress(0.5));
__ j(kBelow, &skip_incr);
__ addsd(t1, codegen_->LiteralDoubleAddress(1.0f));
__ Bind(&skip_incr);
// Final conversion to an integer. Unfortunately this also does not have a // direct x86 instruction, since NaN should map to 0 and large positive // values need to be clipped to the extreme value.
codegen_->Load64BitValue(out, kPrimLongMax);
__ cvtsi2sd(t2, out, /* is64bit= */ true);
__ comisd(t1, t2);
__ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered
__ movl(out, Immediate(0)); // does not change flags, implicit zero extension to 64-bit
__ j(kUnordered, &done); // NaN mapped to 0 (just moved in out)
__ cvttsd2si(out, t1, /* is64bit= */ true);
__ Bind(&done);
}
staticvoid CreateSystemArrayCopyLocations(HInvoke* invoke) {
constexpr int32_t copy_threshold = -1; // negative value means "ignore threshold"
constexpr size_t kInitialNumTemps = 0u; // we need specific temporary register (added below)
LocationSummary* locations = CodeGenerator::CreateSystemArrayCopyLocationSummary(
invoke, copy_threshold, kInitialNumTemps);
if (locations != nullptr) { // And we need some temporaries. We will use REP MOVS{B,W,L}, so we need fixed registers.
locations->AddTemp(Location::CoreRegister(RSI));
locations->AddTemp(Location::CoreRegister(RDI));
locations->AddTemp(Location::CoreRegister(RCX));
}
}
staticvoid CheckSystemArrayCopyPosition(X86_64Assembler* assembler,
CpuRegister array,
Location pos,
Location length,
SlowPathCode* slow_path,
CpuRegister temp, bool length_is_array_length, bool position_sign_checked) { // Where is the length in the Array? const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value();
if (pos.IsConstant()) {
int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); if (pos_const == 0) { if (!length_is_array_length) { // Check that length(array) >= length.
EmitCmplJLess(assembler, Address(array, length_offset), length, slow_path->GetEntryLabel());
}
} else { // Calculate length(array) - pos. // Both operands are known to be non-negative `int32_t`, so the difference cannot underflow // as `int32_t`. If the result is negative, the JL below shall go to the slow path.
__ movl(temp, Address(array, length_offset));
__ subl(temp, Immediate(pos_const));
// Check that (length(array) - pos) >= length.
EmitCmplJLess(assembler, temp, length, slow_path->GetEntryLabel());
}
} elseif (length_is_array_length) { // The only way the copy can succeed is if pos is zero.
CpuRegister pos_reg = pos.AsCoreRegister<CpuRegister>();
__ testl(pos_reg, pos_reg);
__ j(kNotEqual, slow_path->GetEntryLabel());
} else { // Check that pos >= 0.
CpuRegister pos_reg = pos.AsCoreRegister<CpuRegister>(); if (!position_sign_checked) {
__ testl(pos_reg, pos_reg);
__ j(kLess, slow_path->GetEntryLabel());
}
// Calculate length(array) - pos. // Both operands are known to be non-negative `int32_t`, so the difference cannot underflow // as `int32_t`. If the result is negative, the JL below shall go to the slow path.
__ movl(temp, Address(array, length_offset));
__ subl(temp, pos_reg);
// If source and destination are the same, then the copied arrays may overlap. // For overlapping arrays we can only guarantee correctness if `src_pos >= dst_pos`, otherwise // copying the elements at the beginning of source array may clobber the elements at the end. if (!optimizations.GetSourcePositionIsDestinationPosition()) { if (src_pos.IsConstant()) {
int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); if (dest_pos.IsConstant()) {
int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); if (optimizations.GetDestinationIsSource()) { // Checked when building locations.
DCHECK_GE(src_pos_constant, dest_pos_constant);
} elseif (src_pos_constant < dest_pos_constant) {
__ cmpl(src, dest);
__ j(kEqual, slow_path->GetEntryLabel());
}
} else { if (!optimizations.GetDestinationIsSource()) {
__ cmpl(src, dest);
__ j(kNotEqual, &conditions_on_positions_validated);
}
__ cmpl(dest_pos.AsCoreRegister<CpuRegister>(), Immediate(src_pos_constant));
__ j(kGreater, slow_path->GetEntryLabel());
}
} else { if (!optimizations.GetDestinationIsSource()) {
__ cmpl(src, dest);
__ j(kNotEqual, &conditions_on_positions_validated);
}
CpuRegister src_pos_reg = src_pos.AsCoreRegister<CpuRegister>();
EmitCmplJLess(assembler, src_pos_reg, dest_pos, slow_path->GetEntryLabel());
}
}
__ Bind(&conditions_on_positions_validated);
if (!optimizations.GetSourceIsNotNull()) { // Bail out if the source is null.
__ testl(src, src);
__ j(kEqual, slow_path->GetEntryLabel());
}
if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { // Bail out if the destination is null.
__ testl(dest, dest);
__ j(kEqual, slow_path->GetEntryLabel());
}
// If the length is negative, bail out. // We have already checked in the LocationsBuilder for the constant case. if (!length.IsConstant() &&
!optimizations.GetCountIsSourceLength() &&
!optimizations.GetCountIsDestinationLength()) {
__ testl(length.AsCoreRegister<CpuRegister>(), length.AsCoreRegister<CpuRegister>());
__ j(kLess, slow_path->GetEntryLabel());
}
}
// Temporaries that we need for MOVSB/W/L.
CpuRegister src_base = locations->GetTemp(0).AsCoreRegister<CpuRegister>();
DCHECK_EQ(src_base.AsRegister(), RSI);
CpuRegister dest_base = locations->GetTemp(1).AsCoreRegister<CpuRegister>();
DCHECK_EQ(dest_base.AsRegister(), RDI);
CpuRegister count = locations->GetTemp(2).AsCoreRegister<CpuRegister>();
DCHECK_EQ(count.AsRegister(), RCX);
SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen->AddSlowPath(slow_path);
// Check that source and position are different, or if they are the same check that copy // direction is backward. Also check for null pointers.
CheckSystemArrayCopyNullOrOverlap(
invoke, assembler, slow_path, src, dest, src_pos, dest_pos, length);
// Check that source position is within bounds. Use src_base as a temporary register.
CheckSystemArrayCopyPosition(assembler,
src,
src_pos,
length,
slow_path,
src_base, /*length_is_array_length=*/ false, /*position_sign_checked=*/ false);
// Check that destination position is within bounds. Use src_base as a temporary register.
CheckSystemArrayCopyPosition(assembler,
dest,
dest_pos,
length,
slow_path,
src_base, /*length_is_array_length=*/ false, /*position_sign_checked=*/ false);
// We need the count in RCX. if (length.IsConstant()) {
__ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue()));
} else {
__ movl(count, length.AsCoreRegister<CpuRegister>());
}
// Okay, everything checks out. Finally time to do the copy. // Check assumption that sizeof(Char) is 2 (used in scaling below). const size_t data_size = DataType::Size(type); const uint32_t data_offset = mirror::Array::DataOffset(data_size).Uint32Value();
// Do the move. switch (type) { case DataType::Type::kInt8:
__ rep_movsb(); break; case DataType::Type::kUint16:
__ rep_movsw(); break; case DataType::Type::kInt32:
__ rep_movsl(); break; default:
LOG(FATAL) << "Unexpected data type for intrinsic";
}
__ Bind(slow_path->GetExitLabel());
}
void IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopy(HInvoke* invoke) { // The only read barrier implementation supporting the // SystemArrayCopy intrinsic is the Baker-style read barriers. if (codegen_->EmitNonBakerReadBarrier()) { return;
}
constexpr int32_t kLengthThreshold = -1; // No cut-off - handle large arrays in intrinsic code.
constexpr size_t kInitialNumTemps = 0u; // We shall allocate temps explicitly.
LocationSummary* locations = CodeGenerator::CreateSystemArrayCopyLocationSummary(
invoke, kLengthThreshold, kInitialNumTemps); if (locations != nullptr) { // Add temporaries. We will use REP MOVSL, so we need fixed registers.
DCHECK_EQ(locations->GetTempCount(), kInitialNumTemps);
locations->AddTemp(Location::CoreRegister(RSI));
locations->AddTemp(Location::CoreRegister(RDI));
locations->AddTemp(Location::CoreRegister(RCX));
}
}
void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) { // The only read barrier implementation supporting the // SystemArrayCopy intrinsic is the Baker-style read barriers.
DCHECK_IMPLIES(codegen_->EmitReadBarrier(), kUseBakerReadBarrier);
SlowPathCode* intrinsic_slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen_->AddSlowPath(intrinsic_slow_path);
// Check that source and position are different, or if they are the same check that copy // direction is backward. Also check for null pointers.
CheckSystemArrayCopyNullOrOverlap(
invoke, assembler, intrinsic_slow_path, src, dest, src_pos, dest_pos, length);
// Check that source position is within bounds.
CheckSystemArrayCopyPosition(assembler,
src,
src_pos,
length,
intrinsic_slow_path,
temp1,
optimizations.GetCountIsSourceLength(), /*position_sign_checked=*/ false);
// Check that destination position is within bounds. bool dest_position_sign_checked = optimizations.GetSourcePositionIsDestinationPosition();
CheckSystemArrayCopyPosition(assembler,
dest,
dest_pos,
length,
intrinsic_slow_path,
temp1,
optimizations.GetCountIsDestinationLength(),
dest_position_sign_checked);
auto check_non_primitive_array_class = [&](CpuRegister klass, CpuRegister temp) { // No read barrier is needed for reading a chain of constant references for comparing // with null, or for reading a constant primitive value, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp = klass->component_type_
__ movl(temp, Address(klass, component_offset));
__ MaybeUnpoisonHeapReference(temp); // Check that the component type is not null.
__ testl(temp, temp);
__ j(kEqual, intrinsic_slow_path->GetEntryLabel()); // Check that the component type is not a primitive.
__ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
__ j(kNotEqual, intrinsic_slow_path->GetEntryLabel());
};
if (!optimizations.GetDoesNotNeedTypeCheck()) { // Check whether all elements of the source array are assignable to the component // type of the destination array. We do two checks: the classes are the same, // or the destination is Object[]. If none of these checks succeed, we go to the // slow path.
if (codegen_->EmitBakerReadBarrier()) { // /* HeapReference<Class> */ temp1 = dest->klass_
codegen_->GenerateFieldLoadWithBakerReadBarrier(
invoke, temp1_loc, dest, class_offset, /* needs_null_check= */ false); // Register `temp1` is not trashed by the read barrier emitted // by GenerateFieldLoadWithBakerReadBarrier below, as that // method produces a call to a ReadBarrierMarkRegX entry point, // which saves all potentially live registers, including // temporaries such a `temp1`. // /* HeapReference<Class> */ temp2 = src->klass_
codegen_->GenerateFieldLoadWithBakerReadBarrier(
invoke, temp2_loc, src, class_offset, /* needs_null_check= */ false); // If heap poisoning is enabled, `temp1` and `temp2` have been unpoisoned // by the previous calls to GenerateFieldLoadWithBakerReadBarrier.
} else { // /* HeapReference<Class> */ temp1 = dest->klass_
__ movl(temp1, Address(dest, class_offset));
__ MaybeUnpoisonHeapReference(temp1); // /* HeapReference<Class> */ temp2 = src->klass_
__ movl(temp2, Address(src, class_offset));
__ MaybeUnpoisonHeapReference(temp2);
}
__ cmpl(temp1, temp2); if (optimizations.GetDestinationIsTypedObjectArray()) {
DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
NearLabel do_copy; // For class match, we can skip the source type check regardless of the optimization flag.
__ j(kEqual, &do_copy); // No read barrier is needed for reading a chain of constant references // for comparing with null, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp1 = temp1->component_type_
__ movl(temp1, Address(temp1, component_offset));
__ MaybeUnpoisonHeapReference(temp1); // No need to unpoison the following heap reference load, as // we're comparing against null.
__ cmpl(Address(temp1, super_offset), Immediate(0));
__ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); // Bail out if the source is not a non primitive array. if (!optimizations.GetSourceIsNonPrimitiveArray()) {
check_non_primitive_array_class(temp2, CpuRegister(TMP));
}
__ Bind(&do_copy);
} else {
DCHECK(!optimizations.GetDestinationIsTypedObjectArray()); // For class match, we can skip the array type check completely if at least one of source // and destination is known to be a non primitive array, otherwise one check is enough.
__ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); if (!optimizations.GetDestinationIsNonPrimitiveArray() &&
!optimizations.GetSourceIsNonPrimitiveArray()) {
check_non_primitive_array_class(temp2, CpuRegister(TMP));
}
}
} elseif (!optimizations.GetSourceIsNonPrimitiveArray()) {
DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); // Bail out if the source is not a non primitive array. // No read barrier is needed for reading a chain of constant references for comparing // with null, or for reading a constant primitive value, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp1 = src->klass_
__ movl(temp1, Address(src, class_offset));
__ MaybeUnpoisonHeapReference(temp1);
check_non_primitive_array_class(temp1, CpuRegister(TMP));
}
if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) { // Null constant length: not need to emit the loop code at all.
} else { const DataType::Type type = DataType::Type::kReference; const int32_t element_size = DataType::Size(type); const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
// Don't enter copy loop if `length == 0`.
NearLabel skip_copy_and_write_barrier; if (!length.IsConstant()) {
__ testl(length.AsCoreRegister<CpuRegister>(), length.AsCoreRegister<CpuRegister>());
__ j(kEqual, &skip_copy_and_write_barrier);
}
// Compute base source address, base destination address, and end // source address in `temp1`, `temp2` and `temp3` respectively.
GenArrayAddress(assembler, temp1, src, src_pos, type, data_offset);
GenArrayAddress(assembler, temp2, dest, dest_pos, type, data_offset);
SlowPathCode* read_barrier_slow_path = nullptr; if (codegen_->EmitBakerReadBarrier()) { // SystemArrayCopy implementation for Baker read barriers (see // also CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier): // // if (src_ptr != end_ptr) { // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // do { // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++))); // } while (src_ptr != end_ptr) // } else { // // Fast-path copy. // do { // *dest_ptr++ = *src_ptr++; // } while (src_ptr != end_ptr) // } // }
// Given the numeric representation, it's enough to check the low bit of the rb_state.
static_assert(ReadBarrier::NonGrayState() == 0, "Expecting non-gray to have value 0");
static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte;
constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte;
constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position);
// if (rb_state == ReadBarrier::GrayState()) // goto slow_path; // At this point, just do the "if" and make sure that flags are preserved until the branch.
__ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value));
// Load fence to prevent load-load reordering. // Note that this is a no-op, thanks to the x86-64 memory model.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
// Slow path used to copy array when `src` is gray.
read_barrier_slow_path = new (codegen_->GetScopedAllocator()) ReadBarrierSystemArrayCopySlowPathX86_64(invoke);
codegen_->AddSlowPath(read_barrier_slow_path);
// We have done the "if" of the gray bit check above, now branch based on the flags.
__ j(kNotZero, read_barrier_slow_path->GetEntryLabel());
}
// Iterate over the arrays and do a raw copy of the objects. We don't need to poison/unpoison.
DCHECK_EQ(temp1.AsRegister(), RSI);
DCHECK_EQ(temp2.AsRegister(), RDI);
DCHECK_EQ(temp3.AsRegister(), RCX);
__ rep_movsl();
if (read_barrier_slow_path != nullptr) {
DCHECK(codegen_->EmitBakerReadBarrier());
__ Bind(read_barrier_slow_path->GetExitLabel());
}
// We only need one card marking on the destination array.
codegen_->MarkGCCard(temp1, temp2, dest);
// Get offsets of count, value, and class fields within a string object. const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value();
// Note that the null check must have been done earlier.
DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
StringEqualsOptimizations optimizations(invoke); if (!optimizations.GetArgumentNotNull()) { // Check if input is null, return false if it is.
__ testl(arg, arg);
__ j(kEqual, &return_false);
}
if (!optimizations.GetArgumentIsString()) { // Instanceof check for the argument by comparing class fields. // All string objects must have the same type since String cannot be subclassed. // Receiver must be a string object, so its class field is equal to all strings' class fields. // If the argument is a string object, its class field must be equal to receiver's class field. // // As the String class is expected to be non-movable, we can read the class // field from String.equals' arguments without read barriers.
AssertNonMovableStringClass(); // Also, because we use the loaded class references only to compare them, we // don't need to unpoison them. // /* HeapReference<Class> */ rcx = str->klass_
__ movl(rcx, Address(str, class_offset)); // if (rcx != /* HeapReference<Class> */ arg->klass_) return false
__ cmpl(rcx, Address(arg, class_offset));
__ j(kNotEqual, &return_false);
}
// Reference equality check, return true if same reference.
__ cmpl(str, arg);
__ j(kEqual, &return_true);
// Load length and compression flag of receiver string.
__ movl(rcx, Address(str, count_offset)); // Check if lengths and compressiond flags are equal, return false if they're not. // Two identical strings will always have same compression style since // compression style is decided on alloc.
__ cmpl(rcx, Address(arg, count_offset));
__ j(kNotEqual, &return_false); // Return true if both strings are empty. Even with string compression `count == 0` means empty.
static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed");
__ jrcxz(&return_true);
if (mirror::kUseStringCompression) {
NearLabel string_uncompressed; // Extract length and differentiate between both compressed or both uncompressed. // Different compression style is cut above.
__ shrl(rcx, Immediate(1));
__ j(kCarrySet, &string_uncompressed); // Divide string length by 2, rounding up, and continue as if uncompressed. // Merge clearing the compression flag with +1 for rounding.
__ addl(rcx, Immediate(1));
__ shrl(rcx, Immediate(1));
__ Bind(&string_uncompressed);
} // Load starting addresses of string values into RSI/RDI as required for repe_cmpsq instruction.
__ leal(rsi, Address(str, value_offset));
__ leal(rdi, Address(arg, value_offset));
// Divide string length by 4 and adjust for lengths not divisible by 4.
__ addl(rcx, Immediate(3));
__ shrl(rcx, Immediate(2));
// Assertions that must hold in order to compare strings 4 characters (uncompressed) // or 8 characters (compressed) at a time.
DCHECK_ALIGNED(value_offset, 8);
static_assert(IsAligned<8>(kObjectAlignment), "String is not zero padded");
// Loop to compare strings four characters at a time starting at the beginning of the string.
__ repe_cmpsq(); // If strings are not equal, zero flag will be cleared.
__ j(kNotEqual, &return_false);
// Return true and exit the function. // If loop does not result in returning false, we return true.
__ Bind(&return_true);
__ movl(rsi, Immediate(1));
__ jmp(&end);
// Return false and exit the function.
__ Bind(&return_false);
__ xorl(rsi, rsi);
__ Bind(&end);
}
staticvoid CreateStringIndexOfLocations(HInvoke* invoke,
ArenaAllocator* allocator, bool start_at_zero) {
LocationSummary* locations =
LocationSummary::Create(allocator, invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); // The data needs to be in RDI for scasw. So request that the string is there, anyways.
locations->SetInAt(0, Location::CoreRegister(RDI)); // If we look for a constant char, we'll still have to copy it into RAX. So just request the // allocator to do that, anyways. We can still do the constant check by checking the parameter // of the instruction explicitly. // Note: This works as we don't clobber RAX anywhere.
locations->SetInAt(1, Location::CoreRegister(RAX)); if (!start_at_zero) {
locations->SetInAt(2, Location::RequiresCoreRegister()); // The starting index.
} // As we clobber RDI during execution anyways, also use it as the output.
locations->SetOut(Location::SameAsFirstInput());
// repne scasw uses RCX as the counter.
locations->AddTemp(Location::CoreRegister(RCX)); // Need another temporary to be able to compute the result.
locations->AddTemp(Location::RequiresCoreRegister());
}
// Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, // or directly dispatch for a large constant, or omit slow-path for a small constant or a char.
SlowPathCode* slow_path = nullptr;
HInstruction* code_point = invoke->InputAt(1); if (code_point->IsIntConstant()) { if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) >
std::numeric_limits<uint16_t>::max()) { // Always needs the slow-path. We could directly dispatch to it, but this case should be // rare, so for simplicity just put the full slow-path down and branch unconditionally.
slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen->AddSlowPath(slow_path);
__ jmp(slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel()); return;
}
} elseif (code_point->GetType() != DataType::Type::kUint16) {
__ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max()));
slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen->AddSlowPath(slow_path);
__ j(kAbove, slow_path->GetEntryLabel());
}
// From here down, we know that we are looking for a char that fits in // 16 bits (uncompressed) or 8 bits (compressed). // Location of reference to data array within the String object.
int32_t value_offset = mirror::String::ValueOffset().Int32Value(); // Location of count within the String object.
int32_t count_offset = mirror::String::CountOffset().Int32Value();
// Load the count field of the string containing the length and compression flag.
__ movl(string_length, Address(string_obj, count_offset));
// Do a zero-length check. Even with string compression `count == 0` means empty. // TODO: Support jecxz.
NearLabel not_found_label;
__ testl(string_length, string_length);
__ j(kEqual, ¬_found_label);
if (mirror::kUseStringCompression) { // Use TMP to keep string_length_flagged.
__ movl(CpuRegister(TMP), string_length); // Mask out first bit used as compression flag.
__ shrl(string_length, Immediate(1));
}
if (start_at_zero) { // Number of chars to scan is the same as the string length.
__ movl(counter, string_length); // Move to the start of the string.
__ addq(string_obj, Immediate(value_offset));
} else {
CpuRegister start_index = locations->InAt(2).AsCoreRegister<CpuRegister>();
// Do a start_index check.
__ cmpl(start_index, string_length);
__ j(kGreaterEqual, ¬_found_label);
// Ensure we have a start index >= 0;
__ xorl(counter, counter);
__ cmpl(start_index, Immediate(0));
__ cmov(kGreater, counter, start_index, /* is64bit= */ false); // 32-bit copy is enough.
if (mirror::kUseStringCompression) {
NearLabel modify_counter, offset_uncompressed_label;
__ testl(CpuRegister(TMP), Immediate(1));
__ j(kNotZero, &offset_uncompressed_label);
__ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_1, value_offset));
__ jmp(&modify_counter); // Move to the start of the string: string_obj + value_offset + 2 * start_index.
__ Bind(&offset_uncompressed_label);
__ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset));
__ Bind(&modify_counter);
} else {
__ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset));
} // Now update ecx, the work counter: it's gonna be string.length - start_index.
__ negq(counter); // Needs to be 64-bit negation, as the address computation is 64-bit.
__ leaq(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0));
}
if (mirror::kUseStringCompression) {
NearLabel uncompressed_string_comparison;
NearLabel comparison_done;
__ testl(CpuRegister(TMP), Immediate(1));
__ j(kNotZero, &uncompressed_string_comparison); // Check if RAX (search_value) is ASCII.
__ cmpl(search_value, Immediate(127));
__ j(kGreater, ¬_found_label); // Comparing byte-per-byte.
__ repne_scasb();
__ jmp(&comparison_done); // Everything is set up for repne scasw: // * Comparison address in RDI. // * Counter in ECX.
__ Bind(&uncompressed_string_comparison);
__ repne_scasw();
__ Bind(&comparison_done);
} else {
__ repne_scasw();
} // Did we find a match?
__ j(kNotEqual, ¬_found_label);
// Yes, we matched. Compute the index of the result.
__ subl(string_length, counter);
__ leal(out, Address(string_length, -1));
void IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { // No need to emit code checking whether `locations->InAt(2)` is a null // pointer, as callers of the native method // // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) // // all include a null check on `data` before calling that method.
codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke);
CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>();
}
// And we need some temporaries. We will use REP MOVSW, so we need fixed registers.
locations->AddTemp(Location::CoreRegister(RSI));
locations->AddTemp(Location::CoreRegister(RDI));
locations->AddTemp(Location::CoreRegister(RCX));
}
size_t char_component_size = DataType::Size(DataType::Type::kUint16); // Location of data in char array buffer. const uint32_t data_offset = mirror::Array::DataOffset(char_component_size).Uint32Value(); // Location of char array data in string. const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
// public void getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin);
CpuRegister obj = locations->InAt(0).AsCoreRegister<CpuRegister>();
Location srcBegin = locations->InAt(1); int srcBegin_value =
srcBegin.IsConstant() ? srcBegin.GetConstant()->AsIntConstant()->GetValue() : 0;
CpuRegister srcEnd = locations->InAt(2).AsCoreRegister<CpuRegister>();
CpuRegister dst = locations->InAt(3).AsCoreRegister<CpuRegister>();
CpuRegister dstBegin = locations->InAt(4).AsCoreRegister<CpuRegister>();
// Check assumption that sizeof(Char) is 2 (used in scaling below). const size_t char_size = DataType::Size(DataType::Type::kUint16);
DCHECK_EQ(char_size, 2u);
NearLabel done; // Compute the number of chars (words) to move.
__ movl(CpuRegister(RCX), srcEnd); if (srcBegin.IsConstant()) {
__ subl(CpuRegister(RCX), Immediate(srcBegin_value));
} else {
DCHECK(srcBegin.IsCoreRegister());
__ subl(CpuRegister(RCX), srcBegin.AsCoreRegister<CpuRegister>());
} if (mirror::kUseStringCompression) {
NearLabel copy_uncompressed, copy_loop; const size_t c_char_size = DataType::Size(DataType::Type::kInt8);
DCHECK_EQ(c_char_size, 1u); // Location of count in string. const uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
__ leaq(CpuRegister(RSI),
CodeGeneratorX86_64::ArrayAddress(obj, srcBegin, TIMES_2, value_offset)); // Compute the address of the destination buffer.
__ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); // Do the move.
__ rep_movsw();
__ Bind(&done);
}
staticvoid GenPeek(LocationSummary* locations, DataType::Type size, X86_64Assembler* assembler) {
CpuRegister address = locations->InAt(0).AsCoreRegister<CpuRegister>();
CpuRegister out = locations->Out().AsCoreRegister<CpuRegister>(); // x86 allows unaligned access. We do not have to check the input or use specific instructions // to avoid a SIGBUS. switch (size) { case DataType::Type::kInt8:
__ movsxb(out, Address(address, 0)); break; case DataType::Type::kInt16:
__ movsxw(out, Address(address, 0)); break; case DataType::Type::kInt32:
__ movl(out, Address(address, 0)); break; case DataType::Type::kInt64:
__ movq(out, Address(address, 0)); break; default:
LOG(FATAL) << "Type not recognized for peek: " << size;
UNREACHABLE();
}
}
// We don't care for ordered: it requires an AnyStore barrier, which is already given by the x86 // memory model. staticvoid GenUnsafePut(LocationSummary* locations, DataType::Type type, bool is_volatile,
CodeGeneratorX86_64* codegen) {
X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler());
CpuRegister base = locations->InAt(1).AsCoreRegister<CpuRegister>();
CpuRegister offset = locations->InAt(2).AsCoreRegister<CpuRegister>();
CpuRegister value = locations->InAt(3).AsCoreRegister<CpuRegister>();
if (type == DataType::Type::kReference) { bool value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(locations->GetTemp(0).AsCoreRegister<CpuRegister>(),
locations->GetTemp(1).AsCoreRegister<CpuRegister>(),
base,
value,
value_can_be_null);
}
}
// We don't care for ordered: it requires an AnyStore barrier, which is already given by the x86 // memory model. staticvoid GenUnsafePutAbsolute(LocationSummary* locations,
DataType::Type type, bool is_volatile,
CodeGeneratorX86_64* codegen) {
X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler());
CpuRegister address_reg = locations->InAt(1).AsCoreRegister<CpuRegister>();
Address address = Address(address_reg, 0);
CpuRegister value = locations->InAt(2).AsCoreRegister<CpuRegister>();
// RAX is clobbered in CMPXCHG, but we set it as out so no need to add it as temporary.
locations->SetOut(Location::CoreRegister(RAX));
if (type == DataType::Type::kReference) { // Need two temporaries for MarkGCCard. One is also used for reference poisoning.
locations->AddTemp(Location::RequiresCoreRegister());
locations->AddTemp(Location::RequiresCoreRegister()); if (codegen->EmitReadBarrier()) { // Need three temporaries for GenerateReferenceLoadWithBakerReadBarrier.
DCHECK(kUseBakerReadBarrier);
locations->AddTemp(Location::RequiresCoreRegister());
}
}
}
void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { // The only supported read barrier implementation is the Baker-style read barriers. if (codegen_->EmitNonBakerReadBarrier()) { return;
}
// Convert ZF into the Boolean result. staticinlinevoid GenZFlagToResult(X86_64Assembler* assembler, CpuRegister out) {
__ setcc(kZero, out);
__ movzxb(out, out);
}
// This function assumes that expected value for CMPXCHG and output are in RAX. staticvoid GenCompareAndSetOrExchangeInt(CodeGeneratorX86_64* codegen,
DataType::Type type,
Address field_addr,
Location value, bool is_cmpxchg, bool byte_swap) {
X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler());
InstructionCodeGeneratorX86_64* instr_codegen = codegen->GetInstructionCodegen();
if (byte_swap) {
instr_codegen->Bswap(Location::CoreRegister(RAX), type);
instr_codegen->Bswap(value, type);
}
switch (type) { case DataType::Type::kBool: case DataType::Type::kInt8:
__ LockCmpxchgb(field_addr, value.AsCoreRegister<CpuRegister>()); break; case DataType::Type::kInt16: case DataType::Type::kUint16:
__ LockCmpxchgw(field_addr, value.AsCoreRegister<CpuRegister>()); break; case DataType::Type::kInt32: case DataType::Type::kUint32:
__ LockCmpxchgl(field_addr, value.AsCoreRegister<CpuRegister>()); break; case DataType::Type::kInt64: case DataType::Type::kUint64:
__ LockCmpxchgq(field_addr, value.AsCoreRegister<CpuRegister>()); break; default:
LOG(FATAL) << "Unexpected non-integral CAS type " << type;
} // LOCK CMPXCHG has full barrier semantics, so we don't need barriers here.
if (byte_swap) { // Restore byte order for value.
instr_codegen->Bswap(value, type);
}
CpuRegister rax(RAX); if (is_cmpxchg) { if (byte_swap) {
instr_codegen->Bswap(Location::CoreRegister(RAX), type);
} // Sign-extend or zero-extend the result as necessary. switch (type) { case DataType::Type::kBool:
__ movzxb(rax, rax); break; case DataType::Type::kInt8:
__ movsxb(rax, rax); break; case DataType::Type::kInt16:
__ movsxw(rax, rax); break; case DataType::Type::kUint16:
__ movzxw(rax, rax); break; default: break; // No need to do anything.
}
} else {
GenZFlagToResult(assembler, rax);
}
}
DataType::Type type = is64bit ? DataType::Type::kUint64 : DataType::Type::kUint32;
// Copy `expected` to RAX (required by the CMPXCHG instruction).
codegen->Move(rax_loc, expected);
// Copy value to some other register (ensure it's not RAX).
DCHECK_NE(temp.AsRegister(), RAX);
codegen->Move(temp_loc, value);
if (byte_swap) {
instr_codegen->Bswap(rax_loc, type);
instr_codegen->Bswap(temp_loc, type);
}
if (is64bit) {
__ LockCmpxchgq(field_addr, temp);
} else {
__ LockCmpxchgl(field_addr, temp);
} // LOCK CMPXCHG has full barrier semantics, so we don't need barriers here. // No need to restore byte order for temporary register.
if (is_cmpxchg) { if (byte_swap) {
instr_codegen->Bswap(rax_loc, type);
}
MoveIntToFP(out.AsFpuRegister<XmmRegister>(), CpuRegister(RAX), is64bit, assembler);
} else {
GenZFlagToResult(assembler, out.AsCoreRegister<CpuRegister>());
}
}
// This function assumes that expected value for CMPXCHG and output are in RAX. staticvoid GenCompareAndSetOrExchangeRef(CodeGeneratorX86_64* codegen,
HInvoke* invoke,
CpuRegister base,
CpuRegister offset,
CpuRegister value,
CpuRegister temp1,
CpuRegister temp2,
CpuRegister temp3, bool is_cmpxchg) { // The only supported read barrier implementation is the Baker-style read barriers.
DCHECK_IMPLIES(codegen->EmitReadBarrier(), kUseBakerReadBarrier);
// Mark card for object assuming new value is stored. bool value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(temp1, temp2, base, value, value_can_be_null);
Address field_addr(base, offset, TIMES_1, 0); if (codegen->EmitBakerReadBarrier()) { // Need to make sure the reference stored in the field is a to-space // one before attempting the CAS or the CAS could fail incorrectly.
codegen->GenerateReferenceLoadWithBakerReadBarrier(
invoke,
Location::CoreRegister(temp3.AsRegister()),
base,
field_addr, /* needs_null_check= */ false, /* always_update_field= */ true,
&temp1,
&temp2);
} else { // Nothing to do, the value will be loaded into the out register by CMPXCHG.
}
bool base_equals_value = (base.AsRegister() == value.AsRegister()); Register value_reg = value.AsRegister(); if (kPoisonHeapReferences) { if (base_equals_value) { // If `base` and `value` are the same register location, move `value_reg` to a temporary // register. This way, poisoning `value_reg` won't invalidate `base`.
value_reg = temp1.AsRegister();
__ movl(CpuRegister(value_reg), base);
}
// Check that the register allocator did not assign the location of expected value (RAX) to // `value` nor to `base`, so that heap poisoning (when enabled) works as intended below. // - If `value` were equal to RAX, both references would be poisoned twice, meaning they would // not be poisoned at all, as heap poisoning uses address negation. // - If `base` were equal to RAX, poisoning RAX would invalidate `base`.
DCHECK_NE(RAX, value_reg);
DCHECK_NE(RAX, base.AsRegister());
__ LockCmpxchgl(field_addr, CpuRegister(value_reg)); // LOCK CMPXCHG has full barrier semantics, so we don't need barriers.
if (is_cmpxchg) { // Output is in RAX, so we can rely on CMPXCHG and do nothing.
__ MaybeUnpoisonHeapReference(CpuRegister(RAX));
} else {
GenZFlagToResult(assembler, CpuRegister(RAX));
}
// If heap poisoning is enabled, we need to unpoison the values that were poisoned earlier. if (kPoisonHeapReferences) { if (base_equals_value) { // `value_reg` has been moved to a temporary register, no need to unpoison it.
} else { // Ensure `value` is not RAX, so that unpoisoning the former does not invalidate the latter.
DCHECK_NE(RAX, value_reg);
__ UnpoisonHeapReference(CpuRegister(value_reg));
}
}
}
// In debug mode, return true if all registers are pairwise different. In release mode, do nothing // and always return true. staticbool RegsAreAllDifferent(const std::vector<CpuRegister>& regs) { if (kIsDebugBuild) { for (size_t i = 0; i < regs.size(); ++i) { for (size_t j = 0; j < i; ++j) { if (regs[i].AsRegister() == regs[j].AsRegister()) { returnfalse;
}
}
}
} returntrue;
}
// GenCompareAndSetOrExchange handles all value types and therefore accepts generic locations and // temporary indices that may not correspond to real registers for code paths that do not use them. staticvoid GenCompareAndSetOrExchange(CodeGeneratorX86_64* codegen,
HInvoke* invoke,
DataType::Type type,
CpuRegister base,
CpuRegister offset,
uint32_t temp1_index,
uint32_t temp2_index,
uint32_t temp3_index,
Location new_value,
Location expected,
Location out, bool is_cmpxchg, bool byte_swap) {
LocationSummary* locations = invoke->GetLocations();
Address field_address(base, offset, TIMES_1, 0);
GenCompareAndSetOrExchangeFP(
codegen, field_address, temp, new_value, expected, out, is64bit, is_cmpxchg, byte_swap);
} else { // Both the expected value for CMPXCHG and the output are in RAX.
DCHECK_EQ(RAX, expected.AsCoreRegister<Register>());
DCHECK_EQ(RAX, out.AsCoreRegister<Register>());
void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { // The only supported read barrier implementation is the Baker-style read barriers.
DCHECK_IMPLIES(codegen_->EmitReadBarrier(), kUseBakerReadBarrier);
staticvoid CreateUnsafeGetAndUpdateLocations(ArenaAllocator* allocator,
HInvoke* invoke, const CodeGeneratorX86_64* codegen) { constbool can_call = codegen->EmitReadBarrier() && IsUnsafeGetAndSetReference(invoke);
LocationSummary* locations = LocationSummary::Create(
allocator,
invoke,
can_call ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall,
kIntrinsified); if (can_call && kUseBakerReadBarrier) {
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
}
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresCoreRegister());
locations->SetInAt(2, Location::RequiresCoreRegister()); // Use the same register for both the output and the new value or addend // to take advantage of XCHG or XADD. Arbitrarily pick RAX.
locations->SetInAt(3, Location::CoreRegister(RAX)); // Only set the `out` register if it's needed. In the void case we can still use RAX in the // same manner as it is marked as a temp register. if (invoke->GetType() == DataType::Type::kVoid) {
locations->AddTemp(Location::CoreRegister(RAX));
} else {
locations->SetOut(Location::CoreRegister(RAX));
}
}
void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { // The only supported read barrier implementation is the Baker-style read barriers. if (codegen_->EmitNonBakerReadBarrier()) { return;
}
constbool is_void = invoke->GetType() == DataType::Type::kVoid;
Location rax_loc = Location::CoreRegister(RAX); // We requested RAX to use as a temporary for void methods, as we don't return the value.
DCHECK_IMPLIES(!is_void, locations->Out().Equals(rax_loc));
CpuRegister out_or_temp = rax_loc.AsCoreRegister<CpuRegister>(); // Result.
CpuRegister base = locations->InAt(1).AsCoreRegister<CpuRegister>(); // Object pointer.
CpuRegister offset = locations->InAt(2).AsCoreRegister<CpuRegister>(); // Long offset.
DCHECK_EQ(out_or_temp, locations->InAt(3).AsCoreRegister<CpuRegister>()); // New value or addend.
Address field_address(base, offset, TIMES_1, 0);
// In the void case, we have an extra temp register, which is used to signal the register // allocator that we are clobering RAX. const uint32_t extra_temp = is_void ? 1u : 0u;
DCHECK_EQ(locations->GetTempCount(), 3u + extra_temp);
DCHECK_IMPLIES(is_void, locations->GetTemp(0u).Equals(Location::CoreRegister(RAX)));
if (codegen->EmitReadBarrier()) {
DCHECK(kUseBakerReadBarrier); // Ensure that the field contains a to-space reference.
codegen->GenerateReferenceLoadWithBakerReadBarrier(
invoke,
Location::CoreRegister(temp3.AsRegister()),
base,
field_address, /*needs_null_check=*/ false, /*always_update_field=*/ true,
&temp1,
&temp2);
}
// Mark card for object as a new value shall be stored. bool new_value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(temp1, temp2, base, /*value=*/out_or_temp, new_value_can_be_null);
if (kPoisonHeapReferences) { // Use a temp to avoid poisoning base of the field address, which might happen if `out` // is the same as `base` (for code like `unsafe.getAndSet(obj, offset, obj)`).
__ movl(temp1, out_or_temp);
__ PoisonHeapReference(temp1);
__ xchgl(temp1, field_address); if (!is_void) {
__ UnpoisonHeapReference(temp1);
__ movl(out_or_temp, temp1);
}
} else {
__ xchgl(out_or_temp, field_address);
}
}
}
staticvoid CreateBitCountLocations(
ArenaAllocator* allocator, const CodeGeneratorX86_64* codegen, HInvoke* invoke) { if (!codegen->GetInstructionSetFeatures().HasPopCnt()) { // Do nothing if there is no popcnt support. This results in generating // a call for the intrinsic rather than direct code. return;
}
LocationSummary* locations = LocationSummary::CreateNoCall(allocator, invoke, kIntrinsified);
locations->SetInAt(0, Location::Any());
locations->SetOut(Location::RequiresCoreRegister());
}
if (invoke->InputAt(0)->IsConstant()) { // Evaluate this at compile time.
int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant());
int32_t result = is_long
? POPCOUNT(static_cast<uint64_t>(value))
: POPCOUNT(static_cast<uint32_t>(value));
codegen->Load32BitValue(out, result); return;
}
CpuRegister out = locations->Out().AsCoreRegister<CpuRegister>();
InvokeRuntimeCallingConvention calling_convention;
CpuRegister argument = CpuRegister(calling_convention.GetRegisterAt(0)); auto allocate_instance = [&]() {
codegen_->LoadIntrinsicDeclaringClass(argument, invoke);
codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke);
CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
}; if (invoke->InputAt(0)->IsIntConstant()) {
int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); if (static_cast<uint32_t>(value - info.low) < info.length) { // Just embed the object in the code.
DCHECK_NE(info.value_boot_image_reference, ValueOfInfo::kInvalidReference);
codegen_->LoadBootImageAddress(out, info.value_boot_image_reference);
} else {
DCHECK(locations->CanCall()); // Allocate and initialize a new object. // TODO: If we JIT, we could allocate the boxed value now, and store it in the // JIT object table.
allocate_instance();
Store(assembler, type, Address(out, info.value_offset), Immediate(value));
}
} else {
DCHECK(locations->CanCall());
CpuRegister in = locations->InAt(0).AsCoreRegister<CpuRegister>(); // Check bounds of our cache.
__ leal(out, Address(in, -info.low));
__ cmpl(out, Immediate(info.length));
NearLabel allocate, done;
__ j(kAboveEqual, &allocate); // If the value is within the bounds, load the boxed value directly from the array.
DCHECK_NE(out.AsRegister(), argument.AsRegister());
codegen_->LoadBootImageAddress(argument, info.array_data_boot_image_reference);
static_assert((1u << TIMES_4) == sizeof(mirror::HeapReference<mirror::Object>), "Check heap reference size.");
__ movl(out, Address(argument, out, TIMES_4, 0));
__ MaybeUnpoisonHeapReference(out);
__ jmp(&done);
__ Bind(&allocate); // Otherwise allocate and initialize a new object.
allocate_instance();
Store(assembler, type, Address(out, info.value_offset), in);
__ Bind(&done);
}
}
// Load the java.lang.ref.Reference class, use the output register as a temporary.
codegen_->LoadIntrinsicDeclaringClass(out.AsCoreRegister<CpuRegister>(), invoke);
// Load the value from the field.
uint32_t referent_offset = mirror::Reference::ReferentOffset().Uint32Value(); if (codegen_->EmitBakerReadBarrier()) {
codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
out,
obj.AsCoreRegister<CpuRegister>(),
referent_offset, /*needs_null_check=*/ true); // Note that the fence is a no-op, thanks to the x86-64 memory model.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); // `referent` is volatile.
} else {
__ movl(out.AsCoreRegister<CpuRegister>(),
Address(obj.AsCoreRegister<CpuRegister>(), referent_offset));
codegen_->MaybeRecordImplicitNullCheck(invoke); // Note that the fence is a no-op, thanks to the x86-64 memory model.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); // `referent` is volatile.
codegen_->MaybeGenerateReadBarrierSlow(invoke, out, out, obj, referent_offset);
}
__ Bind(slow_path->GetExitLabel());
}
__ movl(out, Address(obj, referent_offset));
codegen_->MaybeRecordImplicitNullCheck(invoke);
__ MaybeUnpoisonHeapReference(out); // Note that the fence is a no-op, thanks to the x86-64 memory model.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); // `referent` is volatile.
__ cmpl(out, other);
if (codegen_->EmitReadBarrier()) {
DCHECK(kUseBakerReadBarrier);
NearLabel calculate_result;
__ j(kEqual, &calculate_result); // ZF set if taken.
// Check if the loaded reference is null in a way that leaves ZF clear for null.
__ cmpl(out, Immediate(1));
__ j(kBelow, &calculate_result); // ZF clear if taken.
// For correct memory visibility, we need a barrier before loading the lock word // but we already have the barrier emitted for volatile load above which is sufficient.
// Load the lockword and check if it is a forwarding address.
static_assert(LockWord::kStateShift == 30u);
static_assert(LockWord::kStateForwardingAddress == 3u);
__ movl(out, Address(out, monitor_offset));
__ cmpl(out, Immediate(static_cast<int32_t>(0xc0000000)));
__ j(kBelow, &calculate_result); // ZF clear if taken.
// Extract the forwarding address and compare with `other`.
__ shll(out, Immediate(LockWord::kForwardingAddressShift));
__ cmpl(out, other);
__ Bind(&calculate_result);
}
// Convert ZF into the Boolean result.
__ setcc(kEqual, out);
__ movzxb(out, out);
}
// We check if the divisor is zero and bail to the slow path to handle if so. auto* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen->AddSlowPath(slow_path);
// If the object is null, there is no need to check the type if (object_can_be_null) {
__ testl(object, object);
__ j(kZero, &type_matched);
}
// Do not unpoison for in-memory comparison. // We deliberately avoid the read barrier, letting the slow path handle the false negatives.
__ movl(temp, Address(object, class_offset));
__ Bind(&check_type_compatibility);
__ cmpl(temp, type_address);
__ j(kEqual, &type_matched); // Load the super class.
__ MaybeUnpoisonHeapReference(temp);
__ movl(temp, Address(temp, super_class_offset)); // If the super class is null, we reached the root of the hierarchy without a match. // We let the slow path handle uncovered cases (e.g. interfaces).
__ testl(temp, temp);
__ j(kEqual, slow_path->GetEntryLabel());
__ jmp(&check_type_compatibility);
__ Bind(&type_matched);
}
// Check access mode and the primitive type from VarHandle.varType. // Check reference arguments against the VarHandle.varType; for references this is a subclass // check without read barrier, so it can have false negatives which we handle in the slow path. staticvoid GenerateVarHandleAccessModeAndVarTypeChecks(HInvoke* invoke,
CodeGeneratorX86_64* codegen,
VarHandleSlowPathX86_64* slow_path,
DataType::Type type) {
X86_64Assembler* assembler = codegen->GetAssembler();
// Check that the operation is permitted.
__ testl(Address(varhandle, access_mode_bit_mask_offset),
Immediate(1u << static_cast<uint32_t>(access_mode)));
__ j(kZero, slow_path->GetEntryLabel());
// For primitive types, we do not need a read barrier when loading a reference only for loading // constant field through the reference. For reference types, we deliberately avoid the read // barrier, letting the slow path handle the false negatives.
__ movl(temp, Address(varhandle, var_type_offset));
__ MaybeUnpoisonHeapReference(temp);
// Check the varType.primitiveType field against the type we're trying to use.
__ cmpw(Address(temp, primitive_type_offset), Immediate(static_cast<uint16_t>(primitive_type)));
__ j(kNotEqual, slow_path->GetEntryLabel());
if (type == DataType::Type::kReference) { // Check reference arguments against the varType. // False negatives due to varType being an interface or array type // or due to the missing read barrier are handled by the slow path.
size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
uint32_t arguments_start = /* VarHandle object */ 1u + expected_coordinates_count;
uint32_t number_of_arguments = invoke->GetNumberOfArguments(); for (size_t arg_index = arguments_start; arg_index != number_of_arguments; ++arg_index) {
HInstruction* arg = invoke->InputAt(arg_index);
DCHECK_EQ(arg->GetType(), DataType::Type::kReference); if (!arg->IsNullConstant()) {
CpuRegister arg_reg = invoke->GetLocations()->InAt(arg_index).AsCoreRegister<CpuRegister>();
Address type_addr(varhandle, var_type_offset);
GenerateSubTypeObjectCheckNoReadBarrier(codegen, slow_path, arg_reg, temp, type_addr);
}
}
}
}
// Check that the VarHandle references a static field by checking that coordinateType0 == null. // Do not emit read barrier (or unpoison the reference) for comparing to null.
__ cmpl(Address(varhandle, coordinate_type0_offset), Immediate(0));
__ j(kNotEqual, slow_path->GetEntryLabel());
}
// Null-check the object. if (!optimizations.GetSkipObjectNullCheck()) {
__ testl(object, object);
__ j(kZero, slow_path->GetEntryLabel());
}
if (!optimizations.GetUseKnownImageVarHandle()) { // Check that the VarHandle references an instance field by checking that // coordinateType1 == null. coordinateType0 should be not null, but this is handled by the // type compatibility check with the source object's type, which will fail for null.
__ cmpl(Address(varhandle, coordinate_type1_offset), Immediate(0));
__ j(kNotEqual, slow_path->GetEntryLabel());
// Check that the object has the correct type. // We deliberately avoid the read barrier, letting the slow path handle the false negatives.
GenerateSubTypeObjectCheckNoReadBarrier(codegen,
slow_path,
object,
temp,
Address(varhandle, coordinate_type0_offset), /*object_can_be_null=*/ false);
}
}
// Check that the VarHandle references an array, byte array view or ByteBuffer by checking // that coordinateType1 != null. If that's true, coordinateType1 shall be int.class and // coordinateType0 shall not be null but we do not explicitly verify that. // No need for read barrier or unpoisoning of coordinateType1 for comparison with null.
__ cmpl(Address(varhandle, coordinate_type1_offset.Int32Value()), Immediate(0));
__ j(kEqual, slow_path->GetEntryLabel());
// Check object class against componentType0. // // This is an exact check and we defer other cases to the runtime. This includes // conversion to array of superclass references, which is valid but subsequently // requires all update operations to check that the value can indeed be stored. // We do not want to perform such extra checks in the intrinsified code. // // We do this check without read barrier, so there can be false negatives which we // defer to the slow path. There shall be no false negatives for array classes in the // boot image (including Object[] and primitive arrays) because they are non-movable.
__ movl(temp, Address(object, class_offset.Int32Value()));
__ cmpl(temp, Address(varhandle, coordinate_type0_offset.Int32Value()));
__ j(kNotEqual, slow_path->GetEntryLabel());
// Check that the coordinateType0 is an array type. We do not need a read barrier // for loading constant reference fields (or chains of them) for comparison with null, // nor for finally loading a constant primitive field (primitive type) below.
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
__ movl(temp, Address(temp, component_type_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
__ testl(temp, temp);
__ j(kZero, slow_path->GetEntryLabel());
// Check that the array component type matches the primitive type.
Label* slow_path_label; if (primitive_type == Primitive::kPrimNot) {
slow_path_label = slow_path->GetEntryLabel();
} else { // With the exception of `kPrimNot` (handled above), `kPrimByte` and `kPrimBoolean`, // we shall check for a byte array view in the slow path. // The check requires the ByteArrayViewVarHandle.class to be in the boot image, // so we cannot emit that if we're JITting without boot image. bool boot_image_available =
codegen->GetCompilerOptions().IsBootImage() ||
!Runtime::Current()->GetHeap()->GetBootImageSpaces().empty(); bool can_be_view = (DataType::Size(value_type) != 1u) && boot_image_available;
slow_path_label =
can_be_view ? slow_path->GetByteArrayViewCheckLabel() : slow_path->GetEntryLabel();
}
__ cmpw(Address(temp, primitive_type_offset), Immediate(static_cast<uint16_t>(primitive_type)));
__ j(kNotEqual, slow_path_label);
// Check for array index out of bounds.
__ cmpl(index, Address(object, array_length_offset.Int32Value()));
__ j(kAboveEqual, slow_path->GetEntryLabel());
}
VarHandleTarget target; // The temporary allocated for loading the offset.
target.offset = locations->GetTemp(0).AsCoreRegister<CpuRegister>().AsRegister(); // The reference to the object that holds the value to operate on.
target.object = (expected_coordinates_count == 0u)
? locations->GetTemp(1).AsCoreRegister<CpuRegister>().AsRegister()
: locations->InAt(1).AsCoreRegister<CpuRegister>().AsRegister(); return target;
}
if (expected_coordinates_count <= 1u) { if (VarHandleOptimizations(invoke).GetUseKnownImageVarHandle()) {
ScopedObjectAccess soa(Thread::Current());
ArtField* target_field = GetImageVarHandleField(invoke); if (expected_coordinates_count == 0u) {
ObjPtr<mirror::Class> declaring_class = target_field->GetDeclaringClass();
__ movl(CpuRegister(target.object),
Address::Absolute(CodeGeneratorX86_64::kPlaceholder32BitOffset, /*no_rip=*/ false)); if (Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(declaring_class)) {
codegen->RecordBootImageRelRoPatch(CodeGenerator::GetBootImageOffset(declaring_class));
} else {
codegen->RecordBootImageTypePatch(declaring_class->GetDexFile(),
declaring_class->GetDexTypeIndex());
}
}
__ movl(CpuRegister(target.offset), Immediate(target_field->GetOffset().Uint32Value()));
} else { // For static fields, we need to fill the `target.object` with the declaring class, // so we can use `target.object` as temporary for the `ArtField*`. For instance fields, // we do not need the declaring class, so we can forget the `ArtField*` when // we load the `target.offset`, so use the `target.offset` to hold the `ArtField*`.
CpuRegister field((expected_coordinates_count == 0) ? target.object : target.offset);
// The effect of LEA is `target.offset = index * scale + data_offset`.
__ leal(CpuRegister(target.offset), Address(index, scale, data_offset.Int32Value()));
}
}
staticbool HasVarHandleIntrinsicImplementation(HInvoke* invoke, const CodeGeneratorX86_64* codegen) { // The only supported read barrier implementation is the Baker-style read barriers. if (codegen->EmitNonBakerReadBarrier()) { returnfalse;
}
VarHandleOptimizations optimizations(invoke); if (optimizations.GetDoNotIntrinsify()) { returnfalse;
}
size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
DCHECK_LE(expected_coordinates_count, 2u); // Filtered by the `DoNotIntrinsify` flag above. returntrue;
}
locations->SetInAt(0, Location::RequiresCoreRegister()); // Require coordinates in registers. These are the object holding the value // to operate on (except for static fields) and index (for arrays and views). for (size_t i = 0; i != expected_coordinates_count; ++i) {
locations->SetInAt(/* VarHandle object */ 1u + i, Location::RequiresCoreRegister());
}
// Accomodating LocationSummary for underlying invoke-* call. for (uint32_t i = 1; i < number_of_args; ++i) {
locations->SetInAt(i, calling_convention.GetNextLocation(invoke->InputAt(i)->GetType()));
}
// Passing MethodHandle object as the last parameter: accessors implementation rely on it.
DCHECK_EQ(invoke->InputAt(0)->GetType(), DataType::Type::kReference);
Location receiver_mh_loc = calling_convention.GetNextLocation(DataType::Type::kReference);
locations->SetInAt(0, receiver_mh_loc);
if (invoke->AsInvokePolymorphic()->NeedsCallSiteTypeCheck()) { // The last input is MethodType object corresponding to the call-site.
locations->SetInAt(number_of_args, Location::RequiresCoreRegister());
}
locations->AddTemp(Location::RequiresCoreRegister()); // Hidden arg for invoke-interface.
locations->AddTemp(Location::CoreRegister(RAX));
if (!receiver_mh_loc.IsCoreRegister()) {
locations->AddTemp(Location::RequiresCoreRegister());
}
}
if (invoke->AsInvokePolymorphic()->NeedsCallSiteTypeCheck()) {
CpuRegister call_site_type =
locations->InAt(invoke->GetNumberOfArguments()).AsCoreRegister<CpuRegister>();
// Call site should match with MethodHandle's type. if (kPoisonHeapReferences) { // call_site_type should be left intact as it 1) might be in callee-saved register 2) is known // for GC to contain a reference.
__ movl(temp, call_site_type);
__ PoisonHeapReference(temp);
__ cmpl(temp, Address(method_handle, mirror::MethodHandle::MethodTypeOffset()));
__ j(kNotEqual, slow_path->GetEntryLabel());
} else {
__ cmpl(call_site_type, Address(method_handle, mirror::MethodHandle::MethodTypeOffset()));
__ j(kNotEqual, slow_path->GetEntryLabel());
}
}
__ Bind(&method_dispatch); if (invoke->AsInvokePolymorphic()->CanTargetInstanceMethod()) {
CpuRegister receiver = locations->InAt(1).AsCoreRegister<CpuRegister>();
// Receiver shouldn't be null for all the following cases.
__ testl(receiver, receiver);
__ j(kEqual, slow_path->GetEntryLabel());
__ cmpl(method_handle_kind, Immediate(mirror::MethodHandle::Kind::kInvokeDirect)); // No dispatch is needed for invoke-direct.
__ j(kEqual, &execute_target_method);
// Skip virtual dispatch if `method` is private.
__ testl(Address(method, ArtMethod::AccessFlagsOffset()), Immediate(kAccPrivate));
__ j(kNotZero, &execute_target_method);
__ movl(temp, Address(method, ArtMethod::DeclaringClassOffset()));
__ cmpl(temp, Address(receiver, mirror::Object::ClassOffset())); // If method is defined in the receiver's class, execute it as it is.
__ j(kEqual, &execute_target_method);
// MethodIndex is uint16_t.
__ movzxw(temp, Address(method, ArtMethod::MethodIndexOffset()));
// Get IMT index. // Not doing default conflict check as IMT index is set for all method which have // kAccAbstract bit.
__ testl(temp, Immediate(kAccAbstract));
__ j(kZero, &get_imt_index_from_method_index);
// imt_index_ is uint16_t
__ movzxw(temp, Address(method, ArtMethod::ImtIndexOffset()));
__ Jump(&do_imt_dispatch);
__ Bind(&do_imt_dispatch); // Re-using `method` to store receiver class and ImTableEntry.
__ movl(method, Address(receiver, mirror::Object::ClassOffset()));
__ MaybeUnpoisonHeapReference(method);
__ Jump(&execute_target_method);
}
__ Bind(&static_dispatch);
__ cmpl(method_handle_kind, Immediate(mirror::MethodHandle::Kind::kInvokeStatic));
__ j(kNotEqual, slow_path->GetEntryLabel()); // MH's kind is invoke-static. The method can be called directly, hence fall-through.
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAcquire(HInvoke* invoke) { // VarHandleGetAcquire is the same as VarHandleGet on x86-64 due to the x86 memory model.
GenerateVarHandleGet(invoke, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetOpaque(HInvoke* invoke) { // VarHandleGetOpaque is the same as VarHandleGet on x86-64 due to the x86 memory model.
GenerateVarHandleGet(invoke, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetVolatile(HInvoke* invoke) { // VarHandleGetVolatile is the same as VarHandleGet on x86-64 due to the x86 memory model.
GenerateVarHandleGet(invoke, codegen_);
}
switch (invoke->GetIntrinsic()) { case Intrinsics::kVarHandleSetRelease:
codegen->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); break; case Intrinsics::kVarHandleSetVolatile: // setVolatile needs kAnyStore barrier, but HandleFieldSet takes care of that. break; default: // Other intrinsics don't need a barrier. break;
}
// Store the value to the field.
codegen->GetInstructionCodegen()->HandleFieldSet(
invoke,
value_index,
last_temp_index,
value_type,
dst,
CpuRegister(target.object),
is_volatile,
is_atomic, /*value_can_be_null=*/true,
byte_swap, // Value can be null, and this write barrier is not being relied on for other sets.
value_type == DataType::Type::kReference ? WriteBarrierKind::kEmitNotBeingReliedOn :
WriteBarrierKind::kDontEmit);
// setVolatile needs kAnyAny barrier, but HandleFieldSet takes care of that.
if (slow_path != nullptr) {
DCHECK(!byte_swap);
__ Bind(slow_path->GetExitLabel());
}
}
if (DataType::IsFloatingPointType(return_type)) {
locations->SetOut(Location::RequiresFpuRegister());
} else { // Take advantage of the fact that CMPXCHG writes result to RAX.
locations->SetOut(Location::CoreRegister(RAX));
}
if (DataType::IsFloatingPointType(expected_type)) { // RAX is needed to load the expected floating-point value into a register for CMPXCHG.
locations->AddTemp(Location::CoreRegister(RAX)); // Another temporary is needed to load the new floating-point value into a register for CMPXCHG.
locations->AddTemp(Location::RequiresCoreRegister());
} else { // Ensure that expected value is in RAX, as required by CMPXCHG.
locations->SetInAt(expected_value_index, Location::CoreRegister(RAX));
locations->SetInAt(new_value_index, Location::RequiresCoreRegister()); if (expected_type == DataType::Type::kReference) { // Need two temporaries for MarkGCCard.
locations->AddRegisterTemps(2); if (codegen->EmitReadBarrier()) { // Need three temporaries for GenerateReferenceLoadWithBakerReadBarrier.
DCHECK(kUseBakerReadBarrier);
locations->AddTemp(Location::RequiresCoreRegister());
}
} // RAX is clobbered in CMPXCHG, but no need to mark it as temporary as it's the output register.
DCHECK_EQ(RAX, locations->Out().AsCoreRegister<Register>());
}
}
// We are using LOCK CMPXCHG in all cases because there is no CAS equivalent that has weak // failure semantics. LOCK CMPXCHG has full barrier semantics, so we don't need barriers.
if (slow_path != nullptr) {
DCHECK(!byte_swap);
__ Bind(slow_path->GetExitLabel());
}
}
// Get the type from the shorty as the invokes may not return a value.
uint32_t number_of_arguments = invoke->GetNumberOfArguments();
uint32_t new_value_index = number_of_arguments - 1;
DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index);
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid;
DCHECK_IMPLIES(!is_void, return_type == value_type);
if (DataType::IsFloatingPointType(value_type)) { // Only set the `out` register if it's needed. In the void case we don't use `out`. if (!is_void) {
locations->SetOut(Location::RequiresFpuRegister());
} // A temporary is needed to load the new floating-point value into a register for XCHG.
locations->AddTemp(Location::RequiresCoreRegister());
} else {
locations->SetInAt(new_value_index, Location::CoreRegister(RAX)); if (value_type == DataType::Type::kReference) { // Need two temporaries for MarkGCCard.
locations->AddRegisterTemps(2); if (codegen->EmitReadBarrier()) { // Need a third temporary for GenerateReferenceLoadWithBakerReadBarrier.
DCHECK(kUseBakerReadBarrier);
locations->AddTemp(Location::RequiresCoreRegister());
}
} // Only set the `out` register if it's needed. In the void case we can still use RAX in the // same manner as it is marked as a temp register. if (is_void) {
locations->AddTemp(Location::CoreRegister(RAX));
} else { // Use the same register for both the new value and output to take advantage of XCHG. // It doesn't have to be RAX, but we need to choose some to make sure it's the same.
locations->SetOut(Location::CoreRegister(RAX));
}
}
}
if (DataType::IsFloatingPointType(type)) { // `getAndSet` for floating-point types: move the new FP value into a register, atomically // exchange it with the field, and move the old value into the output FP register.
Location temp = locations->GetTemp(temp_count - 1);
codegen->Move(temp, value); bool is64bit = (type == DataType::Type::kFloat64);
DataType::Type bswap_type = is64bit ? DataType::Type::kUint64 : DataType::Type::kUint32; if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(temp, bswap_type);
} if (is64bit) {
__ xchgq(temp.AsCoreRegister<CpuRegister>(), field_addr);
} else {
__ xchgl(temp.AsCoreRegister<CpuRegister>(), field_addr);
} if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(temp, bswap_type);
} if (!is_void) {
MoveIntToFP(
out.AsFpuRegister<XmmRegister>(), temp.AsCoreRegister<CpuRegister>(), is64bit, assembler);
}
} elseif (type == DataType::Type::kReference) { // `getAndSet` for references: load reference and atomically exchange it with the field. // Output register is the same as the one holding new value, so no need to move the result.
DCHECK(!byte_swap);
// In the void case, we have an extra temp register, which is used to signal the register // allocator that we are clobering RAX. const uint32_t extra_temp = is_void ? 1u : 0u;
DCHECK_IMPLIES(is_void,
locations->GetTemp(temp_count - 1u).Equals(Location::CoreRegister(RAX)));
DCHECK_IMPLIES(!is_void, valreg == out.AsCoreRegister<CpuRegister>()); if (kPoisonHeapReferences) { // Use a temp to avoid poisoning base of the field address, which might happen if `valreg` is // the same as `target.object` (for code like `vh.getAndSet(obj, obj)`).
__ movl(temp1, valreg);
__ PoisonHeapReference(temp1);
__ xchgl(temp1, field_addr); if (!is_void) {
__ UnpoisonHeapReference(temp1);
__ movl(valreg, temp1);
}
} else {
__ xchgl(valreg, field_addr);
}
} else { // `getAndSet` for integral types: atomically exchange the new value with the field. Output // register is the same as the one holding new value. Do sign extend / zero extend as needed. if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(value, type);
}
CpuRegister valreg = value.AsCoreRegister<CpuRegister>();
DCHECK_IMPLIES(!is_void, valreg == out.AsCoreRegister<CpuRegister>()); switch (type) { case DataType::Type::kBool: case DataType::Type::kUint8:
__ xchgb(valreg, field_addr); if (!is_void) {
__ movzxb(valreg, valreg);
} break; case DataType::Type::kInt8:
__ xchgb(valreg, field_addr); if (!is_void) {
__ movsxb(valreg, valreg);
} break; case DataType::Type::kUint16:
__ xchgw(valreg, field_addr); if (!is_void) {
__ movzxw(valreg, valreg);
} break; case DataType::Type::kInt16:
__ xchgw(valreg, field_addr); if (!is_void) {
__ movsxw(valreg, valreg);
} break; case DataType::Type::kInt32: case DataType::Type::kUint32:
__ xchgl(valreg, field_addr); break; case DataType::Type::kInt64: case DataType::Type::kUint64:
__ xchgq(valreg, field_addr); break; default:
LOG(FATAL) << "unexpected type in getAndSet intrinsic: " << type;
UNREACHABLE();
} if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(value, type);
}
}
}
// Get the type from the shorty as the invokes may not return a value.
uint32_t number_of_arguments = invoke->GetNumberOfArguments();
uint32_t new_value_index = number_of_arguments - 1;
DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index);
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid;
DCHECK_IMPLIES(!is_void, return_type == value_type);
DCHECK_NE(DataType::Type::kReference, value_type);
DCHECK(!DataType::IsFloatingPointType(value_type)); // A temporary to compute the bitwise operation on the old and the new values.
locations->AddTemp(Location::RequiresCoreRegister()); // We need value to be either in a register, or a 32-bit constant (as there are no arithmetic // instructions that accept 64-bit immediate on x86_64).
locations->SetInAt(new_value_index,
DataType::Is64BitType(value_type) ?
Location::RequiresCoreRegister() :
Location::RegisterOrConstant(invoke->InputAt(new_value_index))); if (is_void) { // Used as a temporary, even when we are not outputting it so reserve it. This has to be // requested before the other temporary since there's variable number of temp registers and the // other temp register is expected to be the last one.
locations->AddTemp(Location::CoreRegister(RAX));
} else { // Output is in RAX to accommodate CMPXCHG. It is also used as a temporary.
locations->SetOut(Location::CoreRegister(RAX));
}
}
staticvoid GenerateVarHandleGetAndOp(HInvoke* invoke,
CodeGeneratorX86_64* codegen,
Location value,
DataType::Type type,
Address field_addr,
GetAndUpdateOp get_and_update_op, bool byte_swap) {
X86_64Assembler* assembler = codegen->GetAssembler();
LocationSummary* locations = invoke->GetLocations(); // In the void case, we have an extra temp register, which is used to signal the register // allocator that we are clobering RAX. constbool is_void = invoke->GetType() == DataType::Type::kVoid; const uint32_t extra_temp = is_void ? 1u : 0u; const uint32_t temp_count = locations->GetTempCount();
DCHECK_IMPLIES(is_void,
locations->GetTemp(temp_count - 1u).Equals(Location::CoreRegister(RAX)));
Location temp_loc = locations->GetTemp(temp_count - extra_temp - 1u);
Location rax_loc = Location::CoreRegister(RAX);
DCHECK_IMPLIES(!is_void, locations->Out().Equals(rax_loc));
CpuRegister temp = temp_loc.AsCoreRegister<CpuRegister>(); bool is64Bit = DataType::Is64BitType(type);
NearLabel retry;
__ Bind(&retry);
// Load field value into RAX and copy it into a temporary register for the operation.
codegen->LoadFromMemoryNoReference(type, rax_loc, field_addr);
codegen->Move(temp_loc, rax_loc); if (byte_swap) { // Byte swap the temporary, since we need to perform operation in native endianness.
codegen->GetInstructionCodegen()->Bswap(temp_loc, type);
}
// Use 32-bit registers for 8/16/32-bit types to save on the REX prefix. switch (get_and_update_op) { case GetAndUpdateOp::kAdd:
DCHECK(byte_swap); // The non-byte-swapping path should use a faster XADD instruction. if (is64Bit) {
__ addq(temp, value.AsCoreRegister<CpuRegister>());
} elseif (value.IsConstant()) {
__ addl(temp, Immediate(const_value));
} else {
__ addl(temp, value.AsCoreRegister<CpuRegister>());
} break; case GetAndUpdateOp::kBitwiseAnd: if (is64Bit) {
__ andq(temp, value.AsCoreRegister<CpuRegister>());
} elseif (value.IsConstant()) {
__ andl(temp, Immediate(const_value));
} else {
__ andl(temp, value.AsCoreRegister<CpuRegister>());
} break; case GetAndUpdateOp::kBitwiseOr: if (is64Bit) {
__ orq(temp, value.AsCoreRegister<CpuRegister>());
} elseif (value.IsConstant()) {
__ orl(temp, Immediate(const_value));
} else {
__ orl(temp, value.AsCoreRegister<CpuRegister>());
} break; case GetAndUpdateOp::kBitwiseXor: if (is64Bit) {
__ xorq(temp, value.AsCoreRegister<CpuRegister>());
} elseif (value.IsConstant()) {
__ xorl(temp, Immediate(const_value));
} else {
__ xorl(temp, value.AsCoreRegister<CpuRegister>());
} break; default:
LOG(FATAL) << "unexpected operation";
UNREACHABLE();
}
if (byte_swap) { // RAX still contains the original value, but we need to byte swap the temporary back.
codegen->GetInstructionCodegen()->Bswap(temp_loc, type);
}
switch (type) { case DataType::Type::kBool: case DataType::Type::kUint8: case DataType::Type::kInt8:
__ LockCmpxchgb(field_addr, temp); break; case DataType::Type::kUint16: case DataType::Type::kInt16:
__ LockCmpxchgw(field_addr, temp); break; case DataType::Type::kInt32: case DataType::Type::kUint32:
__ LockCmpxchgl(field_addr, temp); break; case DataType::Type::kInt64: case DataType::Type::kUint64:
__ LockCmpxchgq(field_addr, temp); break; default:
LOG(FATAL) << "unexpected type in getAndBitwiseOp intrinsic";
UNREACHABLE();
}
__ j(kNotZero, &retry);
// The result is in RAX after CMPXCHG. Byte swap if necessary, but do not sign/zero extend, // as it has already been done by `LoadFromMemoryNoReference` above (and not altered by CMPXCHG). if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(rax_loc, type);
}
}
// Get the type from the shorty as the invokes may not return a value.
uint32_t number_of_arguments = invoke->GetNumberOfArguments();
uint32_t new_value_index = number_of_arguments - 1;
DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index);
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid;
DCHECK_IMPLIES(!is_void, return_type == value_type);
if (DataType::IsFloatingPointType(value_type)) { // Only set the `out` register if it's needed. In the void case we don't use `out` if (!is_void) {
locations->SetOut(Location::RequiresFpuRegister());
} // Require that the new FP value is in a register (and not a constant) for ADDSS/ADDSD.
locations->SetInAt(new_value_index, Location::RequiresFpuRegister()); // CMPXCHG clobbers RAX.
locations->AddTemp(Location::CoreRegister(RAX)); // An FP temporary to load the old value from the field and perform FP addition.
locations->AddTemp(Location::RequiresFpuRegister()); // A temporary to hold the new value for CMPXCHG.
locations->AddTemp(Location::RequiresCoreRegister());
} else {
DCHECK_NE(value_type, DataType::Type::kReference);
locations->SetInAt(new_value_index, Location::CoreRegister(RAX)); if (GetExpectedVarHandleCoordinatesCount(invoke) == 2) { // For byte array views with non-native endianness we need extra BSWAP operations, so we // cannot use XADD and have to fallback to a generic implementation based on CMPXCH. In that // case we need two temporary registers: one to hold value instead of RAX (which may get // clobbered by repeated CMPXCHG) and one for performing the operation. At compile time we // cannot distinguish this case from arrays or native-endian byte array views.
locations->AddRegisterTemps(2);
} // Only set the `out` register if it's needed. In the void case we can still use RAX in the // same manner as it is marked as a temp register. if (is_void) {
locations->AddTemp(Location::CoreRegister(RAX));
} else { // Use the same register for both the new value and output to take advantage of XADD. // It should be RAX, because the byte-swapping path of GenerateVarHandleGetAndAdd falls // back to GenerateVarHandleGetAndOp that expects out in RAX.
locations->SetOut(Location::CoreRegister(RAX));
}
}
}
if (DataType::IsFloatingPointType(type)) { if (byte_swap) { // This code should never be executed: it is the case of a byte array view (since it requires // a byte swap), and varhandles for byte array views support numeric atomic update access mode // only for int and long, but not for floating-point types (see javadoc comments for // java.lang.invoke.MethodHandles.byteArrayViewVarHandle()). But ART varhandle implementation // for byte array views treats floating-point types them as numeric types in // ByteArrayViewVarHandle::Access(). Terefore we do generate intrinsic code, but it always // fails access mode check at runtime prior to reaching this point. Illegal instruction UD2 // ensures that if control flow gets here by mistake, we will notice.
__ ud2();
}
// `getAndAdd` for floating-point types: load the old FP value into a temporary FP register and // in RAX for CMPXCHG, add the new FP value to the old one, move it to a non-FP temporary for // CMPXCHG and loop until CMPXCHG succeeds. Move the result from RAX to the output FP register. bool is64bit = (type == DataType::Type::kFloat64);
DataType::Type bswap_type = is64bit ? DataType::Type::kUint64 : DataType::Type::kUint32;
XmmRegister fptemp = locations->GetTemp(temp_count - 2).AsFpuRegister<XmmRegister>();
Location rax_loc = Location::CoreRegister(RAX);
Location temp_loc = locations->GetTemp(temp_count - 1);
CpuRegister temp = temp_loc.AsCoreRegister<CpuRegister>();
NearLabel retry;
__ Bind(&retry);
// Read value from memory into an FP register and copy in into RAX. if (is64bit) {
__ movsd(fptemp, field_addr);
} else {
__ movss(fptemp, field_addr);
}
MoveFPToInt(CpuRegister(RAX), fptemp, is64bit, assembler); // If necessary, byte swap RAX and update the value in FP register to also be byte-swapped. if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(rax_loc, bswap_type);
MoveIntToFP(fptemp, CpuRegister(RAX), is64bit, assembler);
} // Perform the FP addition and move it to a temporary register to prepare for CMPXCHG. if (is64bit) {
__ addsd(fptemp, value.AsFpuRegister<XmmRegister>());
} else {
__ addss(fptemp, value.AsFpuRegister<XmmRegister>());
}
MoveFPToInt(temp, fptemp, is64bit, assembler); // If necessary, byte swap RAX before CMPXCHG and the temporary before copying to FP register. if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(temp_loc, bswap_type);
codegen->GetInstructionCodegen()->Bswap(rax_loc, bswap_type);
} if (is64bit) {
__ LockCmpxchgq(field_addr, temp);
} else {
__ LockCmpxchgl(field_addr, temp);
}
__ j(kNotZero, &retry);
// The old value is in RAX, byte swap if necessary. if (byte_swap) {
codegen->GetInstructionCodegen()->Bswap(rax_loc, bswap_type);
} if (!is_void) {
MoveIntToFP(out.AsFpuRegister<XmmRegister>(), CpuRegister(RAX), is64bit, assembler);
}
} else { if (byte_swap) { // We cannot use XADD since we need to byte-swap the old value when reading it from memory, // and then byte-swap the sum before writing it to memory. So fallback to the slower generic // implementation that is also used for bitwise operations. // Move value from RAX to a temporary register, as RAX may get clobbered by repeated CMPXCHG.
DCHECK_EQ(GetExpectedVarHandleCoordinatesCount(invoke), 2u); // In the void case, we have an extra temp register, which is used to signal the register // allocator that we are clobering RAX. const uint32_t extra_temp = is_void ? 1u : 0u;
DCHECK_IMPLIES(is_void,
locations->GetTemp(temp_count - 1u).Equals(Location::CoreRegister(RAX)));
Location temp = locations->GetTemp(temp_count - extra_temp - 2u);
codegen->Move(temp, value);
GenerateVarHandleGetAndOp(
invoke, codegen, temp, type, field_addr, GetAndUpdateOp::kAdd, byte_swap);
} else { // `getAndAdd` for integral types: atomically exchange the new value with the field and add // the old value to the field. Output register is the same as the one holding new value. Do // sign extend / zero extend as needed.
CpuRegister valreg = value.AsCoreRegister<CpuRegister>();
DCHECK_IMPLIES(!is_void, valreg == out.AsCoreRegister<CpuRegister>()); switch (type) { case DataType::Type::kBool: case DataType::Type::kUint8:
__ LockXaddb(field_addr, valreg); if (!is_void) {
__ movzxb(valreg, valreg);
} break; case DataType::Type::kInt8:
__ LockXaddb(field_addr, valreg); if (!is_void) {
__ movsxb(valreg, valreg);
} break; case DataType::Type::kUint16:
__ LockXaddw(field_addr, valreg); if (!is_void) {
__ movzxw(valreg, valreg);
} break; case DataType::Type::kInt16:
__ LockXaddw(field_addr, valreg); if (!is_void) {
__ movsxw(valreg, valreg);
} break; case DataType::Type::kInt32: case DataType::Type::kUint32:
__ LockXaddl(field_addr, valreg); break; case DataType::Type::kInt64: case DataType::Type::kUint64:
__ LockXaddq(field_addr, valreg); break; default:
LOG(FATAL) << "unexpected type in getAndAdd intrinsic";
UNREACHABLE();
}
}
}
}
// Get the type from the shorty as the invokes may not return a value.
uint32_t number_of_arguments = invoke->GetNumberOfArguments();
Location value = locations->InAt(number_of_arguments - 1);
DataType::Type type = GetDataTypeFromShorty(invoke, number_of_arguments - 1);
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { // `getAndSetAcquire` has `getAcquire` + `set` semantics, so it doesn't need any barriers.
GenerateVarHandleGetAndUpdate(invoke,
codegen_,
GetAndUpdateOp::kSet, /*need_any_store_barrier=*/ false, /*need_any_any_barrier=*/ false);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { // `getAndAddAcquire` has `getAcquire` + `set` semantics, so it doesn't need any barriers.
GenerateVarHandleGetAndUpdate(invoke,
codegen_,
GetAndUpdateOp::kAdd, /*need_any_store_barrier=*/ false, /*need_any_any_barrier=*/ false);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { // `getAndBitwiseAndAcquire` has `getAcquire` + `set` semantics, so it doesn't need any barriers.
GenerateVarHandleGetAndUpdate(invoke,
codegen_,
GetAndUpdateOp::kBitwiseAnd, /*need_any_store_barrier=*/ false, /*need_any_any_barrier=*/ false);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { // `getAndBitwiseOrAcquire` has `getAcquire` + `set` semantics, so it doesn't need any barriers.
GenerateVarHandleGetAndUpdate(invoke,
codegen_,
GetAndUpdateOp::kBitwiseOr, /*need_any_store_barrier=*/ false, /*need_any_any_barrier=*/ false);
}
void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { // `getAndBitwiseXorAcquire` has `getAcquire` + `set` semantics, so it doesn't need any barriers.
GenerateVarHandleGetAndUpdate(invoke,
codegen_,
GetAndUpdateOp::kBitwiseXor, /*need_any_store_barrier=*/ false, /*need_any_any_barrier=*/ false);
}
// The main path checked that the coordinateType0 is an array class that matches // the class of the actual coordinate argument but it does not match the value type. // Check if the `varhandle` references a ByteArrayViewVarHandle instance.
codegen->LoadClassRootForIntrinsic(temp, ClassRoot::kJavaLangInvokeByteArrayViewVarHandle);
assembler->MaybePoisonHeapReference(temp);
__ cmpl(temp, Address(varhandle, class_offset.Int32Value()));
__ j(kNotEqual, GetEntryLabel());
// Check for array index out of bounds.
__ movl(temp, Address(object, array_length_offset.Int32Value())); // SUB sets flags in the same way as CMP.
__ subl(temp, index);
__ j(kBelowEqual, GetEntryLabel()); // The difference between index and array length must be enough for the `value_type` size.
__ cmpl(temp, Immediate(size));
__ j(kBelow, GetEntryLabel());
// Construct the target.
__ leal(CpuRegister(target.offset), Address(index, TIMES_1, data_offset.Int32Value()));
// Alignment check. For unaligned access, go to the runtime.
DCHECK(IsPowerOfTwo(size));
__ testl(CpuRegister(target.offset), Immediate(size - 1u));
__ j(kNotZero, GetEntryLabel());
// Byte order check. For native byte order return to the main path. if (access_mode_template == mirror::VarHandle::AccessModeTemplate::kSet &&
IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { // There is no reason to differentiate between native byte order and byte-swap // for setting a zero bit pattern. Just return to the main path.
__ jmp(GetNativeByteOrderLabel()); return;
}
__ cmpb(Address(varhandle, native_byte_order_offset.Int32Value()), Immediate(0));
__ j(kNotEqual, GetNativeByteOrderLabel());
switch (access_mode_template) { case mirror::VarHandle::AccessModeTemplate::kGet:
GenerateVarHandleGet(invoke, codegen, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kSet:
GenerateVarHandleSet(invoke, codegen, is_volatile_, is_atomic_, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kCompareAndSet:
GenerateVarHandleCompareAndSetOrExchange(
invoke, codegen, /*is_cmpxchg=*/ false, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange:
GenerateVarHandleCompareAndSetOrExchange(
invoke, codegen, /*is_cmpxchg=*/ true, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate:
GenerateVarHandleGetAndUpdate(invoke,
codegen,
get_and_update_op_,
need_any_store_barrier_,
need_any_any_barrier_, /*byte_swap=*/ true); break;
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
