/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "jit/CodeGenerator.h"
#include "mozilla/Assertions.h"
#include "mozilla/Casting.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/EndianUtils.h"
#include "mozilla/EnumeratedArray.h"
#include "mozilla/EnumeratedRange.h"
#include "mozilla/EnumSet.h"
#include "mozilla/IntegerTypeTraits.h"
#include "mozilla/Latin1.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/SIMD.h"
#include <limits>
#include <type_traits>
#include <utility>
#include "jslibmath.h"
#include "jsmath.h"
#include "jsnum.h"
#include "builtin/MapObject.h"
#include "builtin/RegExp.h"
#include "builtin/String.h"
#include "irregexp/RegExpTypes.h"
#include "jit/ABIArgGenerator.h"
#include "jit/CompileInfo.h"
#include "jit/InlineScriptTree.h"
#include "jit/Invalidation.h"
#include "jit/IonGenericCallStub.h"
#include "jit/IonIC.h"
#include "jit/IonScript.h"
#include "jit/JitcodeMap.h"
#include "jit/JitFrames.h"
#include "jit/JitRuntime.h"
#include "jit/JitSpewer.h"
#include "jit/JitZone.h"
#include "jit/Linker.h"
#include "jit/MIRGenerator.h"
#include "jit/MoveEmitter.h"
#include "jit/RangeAnalysis.h"
#include "jit/RegExpStubConstants.h"
#include "jit/SafepointIndex.h"
#include "jit/SharedICHelpers.h"
#include "jit/SharedICRegisters.h"
#include "jit/VMFunctions.h"
#include "jit/WarpSnapshot.h"
#include "js/ColumnNumber.h" // JS::LimitedColumnNumberOneOrigin
#include "js/experimental/JitInfo.h" // JSJit{Getter,Setter}CallArgs, JSJitMethodCallArgsTraits, JSJitInfo
#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
#include "js/RegExpFlags.h" // JS::RegExpFlag
#include "js/ScalarType.h" // js::Scalar::Type
#include "proxy/DOMProxy.h"
#include "proxy/ScriptedProxyHandler.h"
#include "util/CheckedArithmetic.h"
#include "util/Unicode.h"
#include "vm/ArrayBufferViewObject.h"
#include "vm/AsyncFunction.h"
#include "vm/AsyncIteration.h"
#include "vm/BuiltinObjectKind.h"
#include "vm/FunctionFlags.h" // js::FunctionFlags
#include "vm/Interpreter.h"
#include "vm/JSAtomUtils.h" // AtomizeString
#include "vm/MatchPairs.h"
#include "vm/RegExpObject.h"
#include "vm/RegExpStatics.h"
#include "vm/StaticStrings.h"
#include "vm/StringObject.h"
#include "vm/StringType.h"
#include "vm/TypedArrayObject.h"
#include "wasm/WasmCodegenConstants.h"
#include "wasm/WasmPI.h"
#include "wasm/WasmValType.h"
#ifdef MOZ_VTUNE
# include
"vtune/VTuneWrapper.h"
#endif
#include "wasm/WasmBinary.h"
#include "wasm/WasmGC.h"
#include "wasm/WasmGcObject.h"
#include "wasm/WasmStubs.h"
#include "builtin/Boolean-inl.h"
#include "jit/MacroAssembler-inl.h"
#include "jit/shared/CodeGenerator-shared-inl.h"
#include "jit/TemplateObject-inl.h"
#include "jit/VMFunctionList-inl.h"
#include "vm/BytecodeUtil-inl.h"
#include "vm/JSScript-inl.h"
#include "wasm/WasmInstance-inl.h"
using namespace js;
using namespace js::jit;
using mozilla::CheckedUint32;
using mozilla::DebugOnly;
using mozilla::FloatingPoint;
using mozilla::NegativeInfinity;
using mozilla::PositiveInfinity;
using JS::ExpandoAndGeneration;
namespace js {
namespace jit {
#ifdef CHECK_OSIPOINT_REGISTERS
template <
class Op>
static void HandleRegisterDump(Op op, MacroAssembler& masm,
LiveRegisterSet liveRegs,
Register activation,
Register scratch) {
const size_t baseOffset = JitActivation::offsetOfRegs();
// Handle live GPRs.
for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
Register reg = *iter;
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
if (reg == activation) {
// To use the original value of the activation register (that's
// now on top of the stack), we need the scratch register.
masm.push(scratch);
masm.loadPtr(Address(masm.getStackPointer(),
sizeof(uintptr_t)), scratch);
op(scratch, dump);
masm.pop(scratch);
}
else {
op(reg, dump);
}
}
// Handle live FPRs.
for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
FloatRegister reg = *iter;
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
op(reg, dump);
}
}
class StoreOp {
MacroAssembler& masm;
public:
explicit StoreOp(MacroAssembler& masm) : masm(masm) {}
void operator()(
Register reg, Address dump) { masm.storePtr(reg, dump); }
void operator()(FloatRegister reg, Address dump) {
if (reg.isDouble()) {
masm.storeDouble(reg, dump);
}
else if (reg.isSingle()) {
masm.storeFloat32(reg, dump);
}
else if (reg.isSimd128()) {
MOZ_CRASH(
"Unexpected case for SIMD");
}
else {
MOZ_CRASH(
"Unexpected register type.");
}
}
};
class VerifyOp {
MacroAssembler& masm;
Label* failure_;
public:
VerifyOp(MacroAssembler& masm, Label* failure)
: masm(masm), failure_(failure) {}
void operator()(
Register reg, Address dump) {
masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
}
void operator()(FloatRegister reg, Address dump) {
if (reg.isDouble()) {
ScratchDoubleScope scratch(masm);
masm.loadDouble(dump, scratch);
masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
}
else if (reg.isSingle()) {
ScratchFloat32Scope scratch(masm);
masm.loadFloat32(dump, scratch);
masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
}
else if (reg.isSimd128()) {
MOZ_CRASH(
"Unexpected case for SIMD");
}
else {
MOZ_CRASH(
"Unexpected register type.");
}
}
};
void CodeGenerator::verifyOsiPointRegs(LSafepoint* safepoint) {
// Ensure the live registers stored by callVM did not change between
// the call and this OsiPoint. Try-catch relies on this invariant.
// Load pointer to the JitActivation in a scratch register.
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
Register scratch = allRegs.takeAny();
masm.push(scratch);
masm.loadJitActivation(scratch);
// If we should not check registers (because the instruction did not call
// into the VM, or a GC happened), we're done.
Label failure, done;
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
// Having more than one VM function call made in one visit function at
// runtime is a sec-ciritcal error, because if we conservatively assume that
// one of the function call can re-enter Ion, then the invalidation process
// will potentially add a call at a random location, by patching the code
// before the return address.
masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
// Set checkRegs to 0, so that we don't try to verify registers after we
// return from this script to the caller.
masm.store32(Imm32(0), checkRegs);
// Ignore clobbered registers. Some instructions (like LValueToInt32) modify
// temps after calling into the VM. This is fine because no other
// instructions (including this OsiPoint) will depend on them. Also
// backtracking can also use the same register for an input and an output.
// These are marked as clobbered and shouldn't get checked.
LiveRegisterSet liveRegs;
liveRegs.set() = RegisterSet::Intersect(
safepoint->liveRegs().set(),
RegisterSet::
Not(safepoint->clobberedRegs().set()));
VerifyOp op(masm, &failure);
HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
masm.jump(&done);
// Do not profile the callWithABI that occurs below. This is to avoid a
// rare corner case that occurs when profiling interacts with itself:
//
// When slow profiling assertions are turned on, FunctionBoundary ops
// (which update the profiler pseudo-stack) may emit a callVM, which
// forces them to have an osi point associated with them. The
// FunctionBoundary for inline function entry is added to the caller's
// graph with a PC from the caller's code, but during codegen it modifies
// Gecko Profiler instrumentation to add the callee as the current top-most
// script. When codegen gets to the OSIPoint, and the callWithABI below is
// emitted, the codegen thinks that the current frame is the callee, but
// the PC it's using from the OSIPoint refers to the caller. This causes
// the profiler instrumentation of the callWithABI below to ASSERT, since
// the script and pc are mismatched. To avoid this, we simply omit
// instrumentation for these callWithABIs.
// Any live register captured by a safepoint (other than temp registers)
// must remain unchanged between the call and the OsiPoint instruction.
masm.bind(&failure);
masm.assumeUnreachable(
"Modified registers between VM call and OsiPoint");
masm.bind(&done);
masm.pop(scratch);
}
bool CodeGenerator::shouldVerifyOsiPointRegs(LSafepoint* safepoint) {
if (!checkOsiPointRegisters) {
return false;
}
if (safepoint->liveRegs().emptyGeneral() &&
safepoint->liveRegs().emptyFloat()) {
return false;
// No registers to check.
}
return true;
}
void CodeGenerator::resetOsiPointRegs(LSafepoint* safepoint) {
if (!shouldVerifyOsiPointRegs(safepoint)) {
return;
}
// Set checkRegs to 0. If we perform a VM call, the instruction
// will set it to 1.
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
Register scratch = allRegs.takeAny();
masm.push(scratch);
masm.loadJitActivation(scratch);
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
masm.store32(Imm32(0), checkRegs);
masm.pop(scratch);
}
static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) {
// Store a copy of all live registers before performing the call.
// When we reach the OsiPoint, we can use this to check nothing
// modified them in the meantime.
// Load pointer to the JitActivation in a scratch register.
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
Register scratch = allRegs.takeAny();
masm.push(scratch);
masm.loadJitActivation(scratch);
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
masm.add32(Imm32(1), checkRegs);
StoreOp op(masm);
HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
masm.pop(scratch);
}
#endif // CHECK_OSIPOINT_REGISTERS
// Before doing any call to Cpp, you should ensure that volatile
// registers are evicted by the register allocator.
void CodeGenerator::callVMInternal(VMFunctionId id, LInstruction* ins) {
TrampolinePtr code = gen->jitRuntime()->getVMWrapper(id);
const VMFunctionData& fun = GetVMFunction(id);
// Stack is:
// ... frame ...
// [args]
#ifdef DEBUG
MOZ_ASSERT(pushedArgs_ == fun.explicitArgs);
pushedArgs_ = 0;
#endif
#ifdef CHECK_OSIPOINT_REGISTERS
if (shouldVerifyOsiPointRegs(ins->safepoint())) {
StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
}
#endif
#ifdef DEBUG
if (ins->mirRaw()) {
MOZ_ASSERT(ins->mirRaw()->isInstruction());
MInstruction* mir = ins->mirRaw()->toInstruction();
MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint());
// If this MIR instruction has an overridden AliasSet, set the JitRuntime's
// disallowArbitraryCode_ flag so we can assert this VMFunction doesn't call
// RunScript. Whitelist MInterruptCheck and MCheckOverRecursed because
// interrupt callbacks can call JS (chrome JS or shell testing functions).
bool isWhitelisted = mir->isInterruptCheck() || mir->isCheckOverRecursed();
if (!mir->hasDefaultAliasSet() && !isWhitelisted) {
const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
masm.move32(Imm32(1), ReturnReg);
masm.store32(ReturnReg, AbsoluteAddress(addr));
}
}
#endif
// Push an exit frame descriptor.
masm.PushFrameDescriptor(FrameType::IonJS);
// Call the wrapper function. The wrapper is in charge to unwind the stack
// when returning from the call. Failures are handled with exceptions based
// on the return value of the C functions. To guard the outcome of the
// returned value, use another LIR instruction.
ensureOsiSpace();
uint32_t callOffset = masm.callJit(code);
markSafepointAt(callOffset, ins);
#ifdef DEBUG
// Reset the disallowArbitraryCode flag after the call.
{
const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
masm.push(ReturnReg);
masm.move32(Imm32(0), ReturnReg);
masm.store32(ReturnReg, AbsoluteAddress(addr));
masm.pop(ReturnReg);
}
#endif
// Pop rest of the exit frame and the arguments left on the stack.
int framePop =
sizeof(ExitFrameLayout) - ExitFrameLayout::bytesPoppedAfterCall();
masm.implicitPop(fun.explicitStackSlots() *
sizeof(
void*) + framePop);
// Stack is:
// ... frame ...
}
template <
typename Fn, Fn fn>
void CodeGenerator::callVM(LInstruction* ins) {
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
callVMInternal(id, ins);
}
// ArgSeq store arguments for OutOfLineCallVM.
//
// OutOfLineCallVM are created with "oolCallVM" function. The third argument of
// this function is an instance of a class which provides a "generate" in charge
// of pushing the argument, with "pushArg", for a VMFunction.
//
// Such list of arguments can be created by using the "ArgList" function which
// creates one instance of "ArgSeq", where the type of the arguments are
// inferred from the type of the arguments.
//
// The list of arguments must be written in the same order as if you were
// calling the function in C++.
//
// Example:
// ArgList(ToRegister(lir->lhs()), ToRegister(lir->rhs()))
template <
typename... ArgTypes>
class ArgSeq {
std::tuple<std::remove_reference_t<ArgTypes>...> args_;
template <std::size_t... ISeq>
inline void generate(CodeGenerator* codegen,
std::index_sequence<ISeq...>)
const {
// Arguments are pushed in reverse order, from last argument to first
// argument.
(codegen->pushArg(std::get<
sizeof...(ISeq) - 1 - ISeq>(args_)), ...);
}
public:
explicit ArgSeq(ArgTypes&&... args)
: args_(std::forward<ArgTypes>(args)...) {}
inline void generate(CodeGenerator* codegen)
const {
generate(codegen, std::index_sequence_for<ArgTypes...>{});
}
#ifdef DEBUG
static constexpr size_t numArgs =
sizeof...(ArgTypes);
#endif
};
template <
typename... ArgTypes>
inline ArgSeq<ArgTypes...> ArgList(ArgTypes&&... args) {
return ArgSeq<ArgTypes...>(std::forward<ArgTypes>(args)...);
}
// Store wrappers, to generate the right move of data after the VM call.
struct StoreNothing {
inline void generate(CodeGenerator* codegen)
const {}
inline LiveRegisterSet clobbered()
const {
return LiveRegisterSet();
// No register gets clobbered
}
};
class StoreRegisterTo {
private:
Register out_;
public:
explicit StoreRegisterTo(
Register out) : out_(out) {}
inline void generate(CodeGenerator* codegen)
const {
// It's okay to use storePointerResultTo here - the VMFunction wrapper
// ensures the upper bytes are zero for bool/int32 return values.
codegen->storePointerResultTo(out_);
}
inline LiveRegisterSet clobbered()
const {
LiveRegisterSet set;
set.add(out_);
return set;
}
};
class StoreFloatRegisterTo {
private:
FloatRegister out_;
public:
explicit StoreFloatRegisterTo(FloatRegister out) : out_(out) {}
inline void generate(CodeGenerator* codegen)
const {
codegen->storeFloatResultTo(out_);
}
inline LiveRegisterSet clobbered()
const {
LiveRegisterSet set;
set.add(out_);
return set;
}
};
template <
typename Output>
class StoreValueTo_ {
private:
Output out_;
public:
explicit StoreValueTo_(
const Output& out) : out_(out) {}
inline void generate(CodeGenerator* codegen)
const {
codegen->storeResultValueTo(out_);
}
inline LiveRegisterSet clobbered()
const {
LiveRegisterSet set;
set.add(out_);
return set;
}
};
template <
typename Output>
StoreValueTo_<Output> StoreValueTo(
const Output& out) {
return StoreValueTo_<Output>(out);
}
template <
typename Fn, Fn fn,
class ArgSeq,
class StoreOutputTo>
class OutOfLineCallVM :
public OutOfLineCodeBase<CodeGenerator> {
private:
LInstruction* lir_;
ArgSeq args_;
StoreOutputTo out_;
public:
OutOfLineCallVM(LInstruction* lir,
const ArgSeq& args,
const StoreOutputTo& out)
: lir_(lir), args_(args), out_(out) {}
void accept(CodeGenerator* codegen) override {
codegen->visitOutOfLineCallVM(
this);
}
LInstruction* lir()
const {
return lir_; }
const ArgSeq& args()
const {
return args_; }
const StoreOutputTo& out()
const {
return out_; }
};
template <
typename Fn, Fn fn,
class ArgSeq,
class StoreOutputTo>
OutOfLineCode* CodeGenerator::oolCallVM(LInstruction* lir,
const ArgSeq& args,
const StoreOutputTo& out) {
MOZ_ASSERT(lir->mirRaw());
MOZ_ASSERT(lir->mirRaw()->isInstruction());
#ifdef DEBUG
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
const VMFunctionData& fun = GetVMFunction(id);
MOZ_ASSERT(fun.explicitArgs == args.numArgs);
MOZ_ASSERT(fun.returnsData() !=
(std::is_same_v<StoreOutputTo, StoreNothing>));
#endif
OutOfLineCode* ool =
new (alloc())
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>(lir, args, out);
addOutOfLineCode(ool, lir->mirRaw()->toInstruction());
return ool;
}
template <
typename Fn, Fn fn,
class ArgSeq,
class StoreOutputTo>
void CodeGenerator::visitOutOfLineCallVM(
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool) {
LInstruction* lir = ool->lir();
#ifdef JS_JITSPEW
JitSpewStart(JitSpew_Codegen,
" # LIR=%s",
lir->opName());
if (
const char* extra = lir->getExtraName()) {
JitSpewCont(JitSpew_Codegen,
":%s", extra);
}
JitSpewFin(JitSpew_Codegen);
#endif
perfSpewer_.recordInstruction(masm, lir);
saveLive(lir);
ool->args().generate(
this);
callVM<Fn, fn>(lir);
ool->out().generate(
this);
restoreLiveIgnore(lir, ool->out().clobbered());
masm.jump(ool->rejoin());
}
class OutOfLineICFallback :
public OutOfLineCodeBase<CodeGenerator> {
private:
LInstruction* lir_;
size_t cacheIndex_;
size_t cacheInfoIndex_;
public:
OutOfLineICFallback(LInstruction* lir, size_t cacheIndex,
size_t cacheInfoIndex)
: lir_(lir), cacheIndex_(cacheIndex), cacheInfoIndex_(cacheInfoIndex) {}
void bind(MacroAssembler* masm) override {
// The binding of the initial jump is done in
// CodeGenerator::visitOutOfLineICFallback.
}
size_t cacheIndex()
const {
return cacheIndex_; }
size_t cacheInfoIndex()
const {
return cacheInfoIndex_; }
LInstruction* lir()
const {
return lir_; }
void accept(CodeGenerator* codegen) override {
codegen->visitOutOfLineICFallback(
this);
}
};
void CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex) {
if (cacheIndex == SIZE_MAX) {
masm.setOOM();
return;
}
DataPtr<IonIC> cache(
this, cacheIndex);
MInstruction* mir = lir->mirRaw()->toInstruction();
cache->setScriptedLocation(mir->block()->info().script(),
mir->resumePoint()->pc());
Register temp = cache->scratchRegisterForEntryJump();
icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
masm.jump(Address(temp, 0));
MOZ_ASSERT(!icInfo_.empty());
OutOfLineICFallback* ool =
new (alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
addOutOfLineCode(ool, mir);
masm.bind(ool->rejoin());
cache->setRejoinOffset(CodeOffset(ool->rejoin()->offset()));
}
void CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool) {
LInstruction* lir = ool->lir();
size_t cacheIndex = ool->cacheIndex();
size_t cacheInfoIndex = ool->cacheInfoIndex();
DataPtr<IonIC> ic(
this, cacheIndex);
// Register the location of the OOL path in the IC.
ic->setFallbackOffset(CodeOffset(masm.currentOffset()));
switch (ic->kind()) {
case CacheKind::GetProp:
case CacheKind::GetElem: {
IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
saveLive(lir);
pushArg(getPropIC->id());
pushArg(getPropIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonGetPropertyIC*,
HandleValue, HandleValue, MutableHandleValue);
callVM<Fn, IonGetPropertyIC::update>(lir);
StoreValueTo(getPropIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::GetPropSuper:
case CacheKind::GetElemSuper: {
IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
saveLive(lir);
pushArg(getPropSuperIC->id());
pushArg(getPropSuperIC->receiver());
pushArg(getPropSuperIC->object());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject,
HandleValue, HandleValue, MutableHandleValue);
callVM<Fn, IonGetPropSuperIC::update>(lir);
StoreValueTo(getPropSuperIC->output()).generate(
this);
restoreLiveIgnore(lir,
StoreValueTo(getPropSuperIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::SetProp:
case CacheKind::SetElem: {
IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
saveLive(lir);
pushArg(setPropIC->rhs());
pushArg(setPropIC->id());
pushArg(setPropIC->object());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonSetPropertyIC*,
HandleObject, HandleValue, HandleValue);
callVM<Fn, IonSetPropertyIC::update>(lir);
restoreLive(lir);
masm.jump(ool->rejoin());
return;
}
case CacheKind::GetName: {
IonGetNameIC* getNameIC = ic->asGetNameIC();
saveLive(lir);
pushArg(getNameIC->environment());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
MutableHandleValue);
callVM<Fn, IonGetNameIC::update>(lir);
StoreValueTo(getNameIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::BindName: {
IonBindNameIC* bindNameIC = ic->asBindNameIC();
saveLive(lir);
pushArg(bindNameIC->environment());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
JSObject* (*)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
callVM<Fn, IonBindNameIC::update>(lir);
StoreRegisterTo(bindNameIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::GetIterator: {
IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
saveLive(lir);
pushArg(getIteratorIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn = JSObject* (*)(JSContext*, HandleScript, IonGetIteratorIC*,
HandleValue);
callVM<Fn, IonGetIteratorIC::update>(lir);
StoreRegisterTo(getIteratorIC->output()).generate(
this);
restoreLiveIgnore(lir,
StoreRegisterTo(getIteratorIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::OptimizeSpreadCall: {
auto* optimizeSpreadCallIC = ic->asOptimizeSpreadCallIC();
saveLive(lir);
pushArg(optimizeSpreadCallIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonOptimizeSpreadCallIC*,
HandleValue, MutableHandleValue);
callVM<Fn, IonOptimizeSpreadCallIC::update>(lir);
StoreValueTo(optimizeSpreadCallIC->output()).generate(
this);
restoreLiveIgnore(
lir, StoreValueTo(optimizeSpreadCallIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::In: {
IonInIC* inIC = ic->asInIC();
saveLive(lir);
pushArg(inIC->object());
pushArg(inIC->key());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonInIC*, HandleValue,
HandleObject,
bool*);
callVM<Fn, IonInIC::update>(lir);
StoreRegisterTo(inIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::HasOwn: {
IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
saveLive(lir);
pushArg(hasOwnIC->id());
pushArg(hasOwnIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue,
HandleValue, int32_t*);
callVM<Fn, IonHasOwnIC::update>(lir);
StoreRegisterTo(hasOwnIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::CheckPrivateField: {
IonCheckPrivateFieldIC* checkPrivateFieldIC = ic->asCheckPrivateFieldIC();
saveLive(lir);
pushArg(checkPrivateFieldIC->id());
pushArg(checkPrivateFieldIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonCheckPrivateFieldIC*,
HandleValue, HandleValue,
bool*);
callVM<Fn, IonCheckPrivateFieldIC::update>(lir);
StoreRegisterTo(checkPrivateFieldIC->output()).generate(
this);
restoreLiveIgnore(
lir, StoreRegisterTo(checkPrivateFieldIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::InstanceOf: {
IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
saveLive(lir);
pushArg(hasInstanceOfIC->rhs());
pushArg(hasInstanceOfIC->lhs());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonInstanceOfIC*,
HandleValue lhs, HandleObject rhs,
bool* res);
callVM<Fn, IonInstanceOfIC::update>(lir);
StoreRegisterTo(hasInstanceOfIC->output()).generate(
this);
restoreLiveIgnore(lir,
StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::UnaryArith: {
IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
saveLive(lir);
pushArg(unaryArithIC->input());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext* cx, HandleScript outerScript,
IonUnaryArithIC* stub, HandleValue val,
MutableHandleValue res);
callVM<Fn, IonUnaryArithIC::update>(lir);
StoreValueTo(unaryArithIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::ToPropertyKey: {
IonToPropertyKeyIC* toPropertyKeyIC = ic->asToPropertyKeyIC();
saveLive(lir);
pushArg(toPropertyKeyIC->input());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext* cx, HandleScript outerScript,
IonToPropertyKeyIC* ic, HandleValue val,
MutableHandleValue res);
callVM<Fn, IonToPropertyKeyIC::update>(lir);
StoreValueTo(toPropertyKeyIC->output()).generate(
this);
restoreLiveIgnore(lir,
StoreValueTo(toPropertyKeyIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::BinaryArith: {
IonBinaryArithIC* binaryArithIC = ic->asBinaryArithIC();
saveLive(lir);
pushArg(binaryArithIC->rhs());
pushArg(binaryArithIC->lhs());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext* cx, HandleScript outerScript,
IonBinaryArithIC* stub, HandleValue lhs,
HandleValue rhs, MutableHandleValue res);
callVM<Fn, IonBinaryArithIC::update>(lir);
StoreValueTo(binaryArithIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreValueTo(binaryArithIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::Compare: {
IonCompareIC* compareIC = ic->asCompareIC();
saveLive(lir);
pushArg(compareIC->rhs());
pushArg(compareIC->lhs());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext* cx, HandleScript outerScript, IonCompareIC* stub,
HandleValue lhs, HandleValue rhs,
bool* res);
callVM<Fn, IonCompareIC::update>(lir);
StoreRegisterTo(compareIC->output()).generate(
this);
restoreLiveIgnore(lir, StoreRegisterTo(compareIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::CloseIter: {
IonCloseIterIC* closeIterIC = ic->asCloseIterIC();
saveLive(lir);
pushArg(closeIterIC->iter());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonCloseIterIC*, HandleObject);
callVM<Fn, IonCloseIterIC::update>(lir);
restoreLive(lir);
masm.jump(ool->rejoin());
return;
}
case CacheKind::OptimizeGetIterator: {
auto* optimizeGetIteratorIC = ic->asOptimizeGetIteratorIC();
saveLive(lir);
pushArg(optimizeGetIteratorIC->value());
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
pushArg(ImmGCPtr(gen->outerInfo().script()));
using Fn =
bool (*)(JSContext*, HandleScript, IonOptimizeGetIteratorIC*,
HandleValue,
bool* res);
callVM<Fn, IonOptimizeGetIteratorIC::update>(lir);
StoreRegisterTo(optimizeGetIteratorIC->output()).generate(
this);
restoreLiveIgnore(
lir, StoreRegisterTo(optimizeGetIteratorIC->output()).clobbered());
masm.jump(ool->rejoin());
return;
}
case CacheKind::Call:
case CacheKind::TypeOf:
case CacheKind::TypeOfEq:
case CacheKind::ToBool:
case CacheKind::LazyConstant:
case CacheKind::NewArray:
case CacheKind::NewObject:
case CacheKind::Lambda:
case CacheKind::GetImport:
MOZ_CRASH(
"Unsupported IC");
}
MOZ_CRASH();
}
StringObject* MNewStringObject::templateObj()
const {
return &templateObj_->as<StringObject>();
}
CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
MacroAssembler* masm)
: CodeGeneratorSpecific(gen, graph, masm),
ionScriptLabels_(gen->alloc()),
ionNurseryObjectLabels_(gen->alloc()),
scriptCounts_(nullptr) {}
CodeGenerator::~CodeGenerator() { js_delete(scriptCounts_); }
void CodeGenerator::visitValueToNumberInt32(LValueToNumberInt32* lir) {
ValueOperand operand = ToValue(lir->input());
Register output = ToRegister(lir->output());
FloatRegister temp = ToFloatRegister(lir->temp0());
Label fails;
masm.convertValueToInt32(operand, temp, output, &fails,
lir->mir()->needsNegativeZeroCheck(),
lir->mir()->conversion());
bailoutFrom(&fails, lir->snapshot());
}
void CodeGenerator::visitValueTruncateToInt32(LValueTruncateToInt32* lir) {
ValueOperand operand = ToValue(lir->input());
Register output = ToRegister(lir->output());
FloatRegister temp = ToFloatRegister(lir->temp0());
Register stringReg = ToRegister(lir->temp1());
auto* oolDouble = oolTruncateDouble(temp, output, lir->mir());
using Fn =
bool (*)(JSContext*, JSString*,
double*);
auto* oolString = oolCallVM<Fn, StringToNumber>(lir, ArgList(stringReg),
StoreFloatRegisterTo(temp));
Label* stringEntry = oolString->entry();
Label* stringRejoin = oolString->rejoin();
Label fails;
masm.truncateValueToInt32(operand, stringEntry, stringRejoin,
oolDouble->entry(), stringReg, temp, output,
&fails);
masm.bind(oolDouble->rejoin());
bailoutFrom(&fails, lir->snapshot());
}
void CodeGenerator::visitValueToDouble(LValueToDouble* lir) {
ValueOperand operand = ToValue(lir->input());
FloatRegister output = ToFloatRegister(lir->output());
Label fail;
masm.convertValueToDouble(operand, output, &fail);
bailoutFrom(&fail, lir->snapshot());
}
void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) {
ValueOperand operand = ToValue(lir->input());
FloatRegister output = ToFloatRegister(lir->output());
Label fail;
masm.convertValueToFloat32(operand, output, &fail);
bailoutFrom(&fail, lir->snapshot());
}
void CodeGenerator::visitValueToFloat16(LValueToFloat16* lir) {
ValueOperand operand = ToValue(lir->input());
Register temp = ToTempRegisterOrInvalid(lir->temp0());
FloatRegister output = ToFloatRegister(lir->output());
LiveRegisterSet volatileRegs;
if (!MacroAssembler::SupportsFloat64To16()) {
volatileRegs = liveVolatileRegs(lir);
}
Label fail;
masm.convertValueToFloat16(operand, output, temp, volatileRegs, &fail);
bailoutFrom(&fail, lir->snapshot());
}
void CodeGenerator::visitValueToBigInt(LValueToBigInt* lir) {
ValueOperand operand = ToValue(lir->input());
Register output = ToRegister(lir->output());
using Fn = BigInt* (*)(JSContext*, HandleValue);
auto* ool =
oolCallVM<Fn, ToBigInt>(lir, ArgList(operand), StoreRegisterTo(output));
Register tag = masm.extractTag(operand, output);
Label notBigInt, done;
masm.branchTestBigInt(Assembler::NotEqual, tag, ¬BigInt);
masm.unboxBigInt(operand, output);
masm.jump(&done);
masm.bind(¬BigInt);
masm.branchTestBoolean(Assembler::Equal, tag, ool->entry());
masm.branchTestString(Assembler::Equal, tag, ool->entry());
// ToBigInt(object) can have side-effects; all other types throw a TypeError.
bailout(lir->snapshot());
masm.bind(ool->rejoin());
masm.bind(&done);
}
void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) {
masm.convertInt32ToDouble(ToRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) {
masm.convertFloat32ToDouble(ToFloatRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) {
masm.convertDoubleToFloat32(ToFloatRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) {
masm.convertInt32ToFloat32(ToRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitDoubleToFloat16(LDoubleToFloat16* lir) {
LiveRegisterSet volatileRegs;
if (!MacroAssembler::SupportsFloat64To16()) {
volatileRegs = liveVolatileRegs(lir);
}
masm.convertDoubleToFloat16(
ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
}
void CodeGenerator::visitDoubleToFloat32ToFloat16(
LDoubleToFloat32ToFloat16* lir) {
masm.convertDoubleToFloat16(
ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
ToRegister(lir->temp0()), ToRegister(lir->temp1()));
}
void CodeGenerator::visitFloat32ToFloat16(LFloat32ToFloat16* lir) {
LiveRegisterSet volatileRegs;
if (!MacroAssembler::SupportsFloat32To16()) {
volatileRegs = liveVolatileRegs(lir);
}
masm.convertFloat32ToFloat16(
ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
}
void CodeGenerator::visitInt32ToFloat16(LInt32ToFloat16* lir) {
LiveRegisterSet volatileRegs;
if (!MacroAssembler::SupportsFloat32To16()) {
volatileRegs = liveVolatileRegs(lir);
}
masm.convertInt32ToFloat16(
ToRegister(lir->input()), ToFloatRegister(lir->output()),
ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
}
void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) {
Label fail;
FloatRegister input = ToFloatRegister(lir->input());
Register output = ToRegister(lir->output());
masm.convertDoubleToInt32(input, output, &fail,
lir->mir()->needsNegativeZeroCheck());
bailoutFrom(&fail, lir->snapshot());
}
void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) {
Label fail;
FloatRegister input = ToFloatRegister(lir->input());
Register output = ToRegister(lir->output());
masm.convertFloat32ToInt32(input, output, &fail,
lir->mir()->needsNegativeZeroCheck());
bailoutFrom(&fail, lir->snapshot());
}
void CodeGenerator::visitInt32ToIntPtr(LInt32ToIntPtr* lir) {
#ifdef JS_64BIT
// This LIR instruction is only used if the input can be negative.
MOZ_ASSERT(lir->mir()->canBeNegative());
Register output = ToRegister(lir->output());
const LAllocation* input = lir->input();
if (input->isRegister()) {
masm.move32SignExtendToPtr(ToRegister(input), output);
}
else {
masm.load32SignExtendToPtr(ToAddress(input), output);
}
#else
MOZ_CRASH(
"Not used on 32-bit platforms");
#endif
}
void CodeGenerator::visitNonNegativeIntPtrToInt32(
LNonNegativeIntPtrToInt32* lir) {
#ifdef JS_64BIT
Register output = ToRegister(lir->output());
MOZ_ASSERT(ToRegister(lir->input()) == output);
Label bail;
masm.guardNonNegativeIntPtrToInt32(output, &bail);
bailoutFrom(&bail, lir->snapshot());
#else
MOZ_CRASH(
"Not used on 32-bit platforms");
#endif
}
void CodeGenerator::visitIntPtrToDouble(LIntPtrToDouble* lir) {
Register input = ToRegister(lir->input());
FloatRegister output = ToFloatRegister(lir->output());
masm.convertIntPtrToDouble(input, output);
}
void CodeGenerator::visitAdjustDataViewLength(LAdjustDataViewLength* lir) {
Register output = ToRegister(lir->output());
MOZ_ASSERT(ToRegister(lir->input()) == output);
uint32_t byteSize = lir->mir()->byteSize();
#ifdef DEBUG
Label ok;
masm.branchTestPtr(Assembler::NotSigned, output, output, &ok);
masm.assumeUnreachable(
"Unexpected negative value in LAdjustDataViewLength");
masm.bind(&ok);
#endif
Label bail;
masm.branchSubPtr(Assembler::
Signed, Imm32(byteSize - 1), output, &bail);
bailoutFrom(&bail, lir->snapshot());
}
void CodeGenerator::emitOOLTestObject(
Register objreg,
Label* ifEmulatesUndefined,
Label* ifDoesntEmulateUndefined,
Register scratch) {
saveVolatile(scratch);
#if defined(DEBUG) ||
defined(FUZZING)
masm.loadPtr(AbsoluteAddress(
gen->runtime->addressOfHasSeenObjectEmulateUndefinedFuse()),
scratch);
using Fn =
bool (*)(JSObject* obj, size_t fuseValue);
masm.setupAlignedABICall();
masm.passABIArg(objreg);
masm.passABIArg(scratch);
masm.callWithABI<Fn, js::EmulatesUndefinedCheckFuse>();
#else
using Fn =
bool (*)(JSObject* obj);
masm.setupAlignedABICall();
masm.passABIArg(objreg);
masm.callWithABI<Fn, js::EmulatesUndefined>();
#endif
masm.storeCallPointerResult(scratch);
restoreVolatile(scratch);
masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
masm.jump(ifDoesntEmulateUndefined);
}
// Base out-of-line code generator for all tests of the truthiness of an
// object, where the object might not be truthy. (Recall that per spec all
// objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
// flag to permit objects to look like |undefined| in certain contexts,
// including in object truthiness testing.) We check truthiness inline except
// when we're testing it on a proxy, in which case out-of-line code will call
// EmulatesUndefined for a conclusive answer.
class OutOfLineTestObject :
public OutOfLineCodeBase<CodeGenerator> {
Register objreg_;
Register scratch_;
Label* ifEmulatesUndefined_;
Label* ifDoesntEmulateUndefined_;
#ifdef DEBUG
bool initialized() {
return ifEmulatesUndefined_ != nullptr; }
#endif
public:
OutOfLineTestObject()
: ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) {}
void accept(CodeGenerator* codegen) final {
MOZ_ASSERT(initialized());
codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_,
ifDoesntEmulateUndefined_, scratch_);
}
// Specify the register where the object to be tested is found, labels to
// jump to if the object is truthy or falsy, and a scratch register for
// use in the out-of-line path.
void setInputAndTargets(
Register objreg, Label* ifEmulatesUndefined,
Label* ifDoesntEmulateUndefined,
Register scratch) {
MOZ_ASSERT(!initialized());
MOZ_ASSERT(ifEmulatesUndefined);
objreg_ = objreg;
scratch_ = scratch;
ifEmulatesUndefined_ = ifEmulatesUndefined;
ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
}
};
// A subclass of OutOfLineTestObject containing two extra labels, for use when
// the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
// code. The user should bind these labels in inline code, and specify them as
// targets via setInputAndTargets, as appropriate.
class OutOfLineTestObjectWithLabels :
public OutOfLineTestObject {
Label label1_;
Label label2_;
public:
OutOfLineTestObjectWithLabels() =
default;
Label* label1() {
return &label1_; }
Label* label2() {
return &label2_; }
};
void CodeGenerator::testObjectEmulatesUndefinedKernel(
Register objreg, Label* ifEmulatesUndefined,
Label* ifDoesntEmulateUndefined,
Register scratch,
OutOfLineTestObject* ool) {
ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
scratch);
// Perform a fast-path check of the object's class flags if the object's
// not a proxy. Let out-of-line code handle the slow cases that require
// saving registers, making a function call, and restoring registers.
masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(),
ifEmulatesUndefined);
}
void CodeGenerator::branchTestObjectEmulatesUndefined(
Register objreg, Label* ifEmulatesUndefined,
Label* ifDoesntEmulateUndefined,
Register scratch,
OutOfLineTestObject* ool) {
MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
"ifDoesntEmulateUndefined will be bound to the fallthrough path");
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
ifDoesntEmulateUndefined, scratch, ool);
masm.bind(ifDoesntEmulateUndefined);
}
void CodeGenerator::testObjectEmulatesUndefined(
Register objreg,
Label* ifEmulatesUndefined,
Label* ifDoesntEmulateUndefined,
Register scratch,
OutOfLineTestObject* ool) {
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
ifDoesntEmulateUndefined, scratch, ool);
masm.jump(ifDoesntEmulateUndefined);
}
void CodeGenerator::testValueTruthyForType(
JSValueType type, ScratchTagScope& tag,
const ValueOperand& value,
Register tempToUnbox,
Register temp, FloatRegister floatTemp,
Label* ifTruthy, Label* ifFalsy, OutOfLineTestObject* ool,
bool skipTypeTest) {
#ifdef DEBUG
if (skipTypeTest) {
Label expected;
masm.branchTestType(Assembler::Equal, tag, type, &expected);
masm.assumeUnreachable(
"Unexpected Value type in testValueTruthyForType");
masm.bind(&expected);
}
#endif
// Handle irregular types first.
switch (type) {
case JSVAL_TYPE_UNDEFINED:
case JSVAL_TYPE_NULL:
// Undefined and null are falsy.
if (!skipTypeTest) {
masm.branchTestType(Assembler::Equal, tag, type, ifFalsy);
}
else {
masm.jump(ifFalsy);
}
return;
case JSVAL_TYPE_SYMBOL:
// Symbols are truthy.
if (!skipTypeTest) {
masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
}
else {
masm.jump(ifTruthy);
}
return;
case JSVAL_TYPE_OBJECT: {
Label notObject;
if (!skipTypeTest) {
masm.branchTestObject(Assembler::NotEqual, tag, ¬Object);
}
ScratchTagScopeRelease _(&tag);
Register objreg = masm.extractObject(value, tempToUnbox);
testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, temp, ool);
masm.bind(¬Object);
return;
}
default:
break;
}
// Check the type of the value (unless this is the last possible type).
Label differentType;
if (!skipTypeTest) {
masm.branchTestType(Assembler::NotEqual, tag, type, &differentType);
}
// Branch if the value is falsy.
ScratchTagScopeRelease _(&tag);
switch (type) {
case JSVAL_TYPE_BOOLEAN: {
masm.branchTestBooleanTruthy(
false, value, ifFalsy);
break;
}
case JSVAL_TYPE_INT32: {
masm.branchTestInt32Truthy(
false, value, ifFalsy);
break;
}
case JSVAL_TYPE_STRING: {
masm.branchTestStringTruthy(
false, value, ifFalsy);
break;
}
case JSVAL_TYPE_BIGINT: {
masm.branchTestBigIntTruthy(
false, value, ifFalsy);
break;
}
case JSVAL_TYPE_DOUBLE: {
masm.unboxDouble(value, floatTemp);
masm.branchTestDoubleTruthy(
false, floatTemp, ifFalsy);
break;
}
default:
MOZ_CRASH(
"Unexpected value type");
}
// If we reach this point, the value is truthy. We fall through for
// truthy on the last test; otherwise, branch.
if (!skipTypeTest) {
masm.jump(ifTruthy);
}
masm.bind(&differentType);
}
void CodeGenerator::testValueTruthy(
const ValueOperand& value,
Register tempToUnbox,
Register temp,
FloatRegister floatTemp,
const TypeDataList& observedTypes,
Label* ifTruthy, Label* ifFalsy,
OutOfLineTestObject* ool) {
ScratchTagScope tag(masm, value);
masm.splitTagForTest(value, tag);
const std::initializer_list<JSValueType> defaultOrder = {
JSVAL_TYPE_UNDEFINED, JSVAL_TYPE_NULL, JSVAL_TYPE_BOOLEAN,
JSVAL_TYPE_INT32, JSVAL_TYPE_OBJECT, JSVAL_TYPE_STRING,
JSVAL_TYPE_DOUBLE, JSVAL_TYPE_SYMBOL, JSVAL_TYPE_BIGINT};
mozilla::EnumSet<JSValueType, uint32_t> remaining(defaultOrder);
// Generate tests for previously observed types first.
// The TypeDataList is sorted by descending frequency.
for (
auto& observed : observedTypes) {
JSValueType type = observed.type();
remaining -= type;
testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
ifTruthy, ifFalsy, ool,
/*skipTypeTest*/ false);
}
// Generate tests for remaining types.
for (
auto type : defaultOrder) {
if (!remaining.contains(type)) {
continue;
}
remaining -= type;
// We don't need a type test for the last possible type.
bool skipTypeTest = remaining.isEmpty();
testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
ifTruthy, ifFalsy, ool, skipTypeTest);
}
MOZ_ASSERT(remaining.isEmpty());
// We fall through if the final test is truthy.
}
void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
Register input = ToRegister(test->input());
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
if (isNextBlock(ifFalse->lir())) {
masm.branchTest32(Assembler::NonZero, input, input,
getJumpLabelForBranch(ifTrue));
}
else {
masm.branchTest32(Assembler::Zero, input, input,
getJumpLabelForBranch(ifFalse));
jumpToBlock(ifTrue);
}
}
void CodeGenerator::visitTestIPtrAndBranch(LTestIPtrAndBranch* test) {
Register input = ToRegister(test->input());
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
if (isNextBlock(ifFalse->lir())) {
masm.branchTestPtr(Assembler::NonZero, input, input,
getJumpLabelForBranch(ifTrue));
}
else {
masm.branchTestPtr(Assembler::Zero, input, input,
getJumpLabelForBranch(ifFalse));
jumpToBlock(ifTrue);
}
}
void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* test) {
Register64 input = ToRegister64(test->input());
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
if (isNextBlock(ifFalse->lir())) {
masm.branchTest64(Assembler::NonZero, input, input,
getJumpLabelForBranch(ifTrue));
}
else if (isNextBlock(ifTrue->lir())) {
masm.branchTest64(Assembler::Zero, input, input,
getJumpLabelForBranch(ifFalse));
}
else {
masm.branchTest64(Assembler::NonZero, input, input,
getJumpLabelForBranch(ifTrue),
getJumpLabelForBranch(ifFalse));
}
}
void CodeGenerator::visitTestBIAndBranch(LTestBIAndBranch* lir) {
Register input = ToRegister(lir->input());
MBasicBlock* ifTrue = lir->ifTrue();
MBasicBlock* ifFalse = lir->ifFalse();
if (isNextBlock(ifFalse->lir())) {
masm.branchIfBigIntIsNonZero(input, getJumpLabelForBranch(ifTrue));
}
else {
masm.branchIfBigIntIsZero(input, getJumpLabelForBranch(ifFalse));
jumpToBlock(ifTrue);
}
}
static Assembler::Condition ReverseCondition(Assembler::Condition condition) {
switch (condition) {
case Assembler::Equal:
case Assembler::NotEqual:
return condition;
case Assembler::Above:
return Assembler::Below;
case Assembler::AboveOrEqual:
return Assembler::BelowOrEqual;
case Assembler::Below:
return Assembler::Above;
case Assembler::BelowOrEqual:
return Assembler::AboveOrEqual;
case Assembler::GreaterThan:
return Assembler::LessThan;
case Assembler::GreaterThanOrEqual:
return Assembler::LessThanOrEqual;
case Assembler::LessThan:
return Assembler::GreaterThan;
case Assembler::LessThanOrEqual:
return Assembler::GreaterThanOrEqual;
default:
break;
}
MOZ_CRASH(
"unhandled condition");
}
void CodeGenerator::visitCompare(LCompare* comp) {
MCompare::CompareType compareType = comp->mir()->compareType();
Assembler::Condition cond = JSOpToCondition(compareType, comp->jsop());
Register left = ToRegister(comp->left());
const LAllocation* right = comp->right();
Register output = ToRegister(comp->output());
if (compareType == MCompare::Compare_Object ||
compareType == MCompare::Compare_Symbol ||
compareType == MCompare::Compare_IntPtr ||
compareType == MCompare::Compare_UIntPtr ||
compareType == MCompare::Compare_WasmAnyRef) {
if (right->isConstant()) {
MOZ_ASSERT(compareType == MCompare::Compare_IntPtr ||
compareType == MCompare::Compare_UIntPtr);
masm.cmpPtrSet(cond, left, ImmWord(ToInt32(right)), output);
}
else if (right->isRegister()) {
masm.cmpPtrSet(cond, left, ToRegister(right), output);
}
else {
masm.cmpPtrSet(ReverseCondition(cond), ToAddress(right), left, output);
}
return;
}
MOZ_ASSERT(compareType == MCompare::Compare_Int32 ||
compareType == MCompare::Compare_UInt32);
if (right->isConstant()) {
masm.cmp32Set(cond, left, Imm32(ToInt32(right)), output);
}
else if (right->isRegister()) {
masm.cmp32Set(cond, left, ToRegister(right), output);
}
else {
masm.cmp32Set(ReverseCondition(cond), ToAddress(right), left, output);
}
}
void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
MCompare::CompareType compareType = comp->cmpMir()->compareType();
Assembler::Condition cond = JSOpToCondition(compareType, comp->jsop());
Register left = ToRegister(comp->left());
const LAllocation* right = comp->right();
MBasicBlock* ifTrue = comp->ifTrue();
MBasicBlock* ifFalse = comp->ifFalse();
// If the next block is the true case, invert the condition to fall through.
Label* label;
if (isNextBlock(ifTrue->lir())) {
cond = Assembler::InvertCondition(cond);
label = getJumpLabelForBranch(ifFalse);
}
else {
label = getJumpLabelForBranch(ifTrue);
}
if (compareType == MCompare::Compare_Object ||
compareType == MCompare::Compare_Symbol ||
compareType == MCompare::Compare_IntPtr ||
compareType == MCompare::Compare_UIntPtr ||
compareType == MCompare::Compare_WasmAnyRef) {
if (right->isConstant()) {
MOZ_ASSERT(compareType == MCompare::Compare_IntPtr ||
compareType == MCompare::Compare_UIntPtr);
masm.branchPtr(cond, left, ImmWord(ToInt32(right)), label);
}
else if (right->isRegister()) {
masm.branchPtr(cond, left, ToRegister(right), label);
}
else {
masm.branchPtr(ReverseCondition(cond), ToAddress(right), left, label);
}
}
else {
MOZ_ASSERT(compareType == MCompare::Compare_Int32 ||
compareType == MCompare::Compare_UInt32);
if (right->isConstant()) {
masm.branch32(cond, left, Imm32(ToInt32(right)), label);
}
else if (right->isRegister()) {
masm.branch32(cond, left, ToRegister(right), label);
}
else {
masm.branch32(ReverseCondition(cond), ToAddress(right), left, label);
}
}
if (!isNextBlock(ifTrue->lir())) {
jumpToBlock(ifFalse);
}
}
void CodeGenerator::visitCompareI64(LCompareI64* lir) {
MCompare::CompareType compareType = lir->mir()->compareType();
MOZ_ASSERT(compareType == MCompare::Compare_Int64 ||
compareType == MCompare::Compare_UInt64);
bool isSigned = compareType == MCompare::Compare_Int64;
Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
Register64 left = ToRegister64(lir->left());
LInt64Allocation right = lir->right();
Register output = ToRegister(lir->output());
if (IsConstant(right)) {
masm.cmp64Set(cond, left, Imm64(ToInt64(right)), output);
}
else if (IsRegister64(right)) {
masm.cmp64Set(cond, left, ToRegister64(right), output);
}
else {
masm.cmp64Set(ReverseCondition(cond), ToAddress(right), left, output);
}
}
void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
MCompare::CompareType compareType = lir->cmpMir()->compareType();
MOZ_ASSERT(compareType == MCompare::Compare_Int64 ||
compareType == MCompare::Compare_UInt64);
bool isSigned = compareType == MCompare::Compare_Int64;
Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
Register64 left = ToRegister64(lir->left());
LInt64Allocation right = lir->right();
MBasicBlock* ifTrue = lir->ifTrue();
MBasicBlock* ifFalse = lir->ifFalse();
Label* trueLabel = getJumpLabelForBranch(ifTrue);
Label* falseLabel = getJumpLabelForBranch(ifFalse);
// If the next block is the true case, invert the condition to fall through.
if (isNextBlock(ifTrue->lir())) {
cond = Assembler::InvertCondition(cond);
trueLabel = falseLabel;
falseLabel = nullptr;
}
else if (isNextBlock(ifFalse->lir())) {
falseLabel = nullptr;
}
if (IsConstant(right)) {
masm.branch64(cond, left, Imm64(ToInt64(right)), trueLabel, falseLabel);
}
else if (IsRegister64(right)) {
masm.branch64(cond, left, ToRegister64(right), trueLabel, falseLabel);
}
else {
masm.branch64(ReverseCondition(cond), ToAddress(right), left, trueLabel,
falseLabel);
}
}
void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
Assembler::Condition cond = baab->cond();
MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
Register left = ToRegister(baab->left());
const LAllocation* right = baab->right();
MBasicBlock* ifTrue = baab->ifTrue();
MBasicBlock* ifFalse = baab->ifFalse();
// If the next block is the true case, invert the condition to fall through.
Label* label;
if (isNextBlock(ifTrue->lir())) {
cond = Assembler::InvertCondition(cond);
label = getJumpLabelForBranch(ifFalse);
}
else {
label = getJumpLabelForBranch(ifTrue);
}
if (right->isConstant()) {
masm.branchTest32(cond, left, Imm32(ToInt32(right)), label);
}
else {
masm.branchTest32(cond, left, ToRegister(right), label);
}
if (!isNextBlock(ifTrue->lir())) {
jumpToBlock(ifFalse);
}
}
void CodeGenerator::visitBitAnd64AndBranch(LBitAnd64AndBranch* baab) {
Assembler::Condition cond = baab->cond();
MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
Register64 left = ToRegister64(baab->left());
LInt64Allocation right = baab->right();
MBasicBlock* ifTrue = baab->ifTrue();
MBasicBlock* ifFalse = baab->ifFalse();
Label* trueLabel = getJumpLabelForBranch(ifTrue);
Label* falseLabel = getJumpLabelForBranch(ifFalse);
// If the next block is the true case, invert the condition to fall through.
if (isNextBlock(ifTrue->lir())) {
cond = Assembler::InvertCondition(cond);
trueLabel = falseLabel;
falseLabel = nullptr;
}
else if (isNextBlock(ifFalse->lir())) {
falseLabel = nullptr;
}
if (IsConstant(right)) {
masm.branchTest64(cond, left, Imm64(ToInt64(right)), trueLabel, falseLabel);
}
else {
masm.branchTest64(cond, left, ToRegister64(right), trueLabel, falseLabel);
}
}
void CodeGenerator::assertObjectDoesNotEmulateUndefined(
Register input,
Register temp,
const MInstruction* mir) {
#if defined(DEBUG) ||
defined(FUZZING)
// Validate that the object indeed doesn't have the emulates undefined flag.
auto* ool =
new (alloc()) OutOfLineTestObjectWithLabels();
addOutOfLineCode(ool, mir);
Label* doesNotEmulateUndefined = ool->label1();
Label* emulatesUndefined = ool->label2();
testObjectEmulatesUndefined(input, emulatesUndefined, doesNotEmulateUndefined,
temp, ool);
masm.bind(emulatesUndefined);
masm.assumeUnreachable(
"Found an object emulating undefined while the fuse is intact");
masm.bind(doesNotEmulateUndefined);
#endif
}
void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) {
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
Register input = ToRegister(lir->input());
Register temp = ToRegister(lir->temp0());
bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
if (intact) {
assertObjectDoesNotEmulateUndefined(input, temp, lir->mir());
// Bug 1874905: It would be fantastic if this could be optimized out
masm.jump(truthy);
}
else {
auto* ool =
new (alloc()) OutOfLineTestObject();
addOutOfLineCode(ool, lir->mir());
testObjectEmulatesUndefined(input, falsy, truthy, temp, ool);
}
}
void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) {
auto* ool =
new (alloc()) OutOfLineTestObject();
addOutOfLineCode(ool, lir->mir());
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
ValueOperand input = ToValue(lir->input());
Register tempToUnbox = ToTempUnboxRegister(lir->temp1());
Register temp = ToRegister(lir->temp2());
FloatRegister floatTemp = ToFloatRegister(lir->temp0());
const TypeDataList& observedTypes = lir->mir()->observedTypes();
testValueTruthy(input, tempToUnbox, temp, floatTemp, observedTypes, truthy,
falsy, ool);
masm.jump(truthy);
}
void CodeGenerator::visitBooleanToString(LBooleanToString* lir) {
Register input = ToRegister(lir->input());
Register output = ToRegister(lir->output());
const JSAtomState& names = gen->runtime->names();
Label true_, done;
masm.branchTest32(Assembler::NonZero, input, input, &true_);
masm.movePtr(ImmGCPtr(names.false_), output);
masm.jump(&done);
masm.bind(&true_);
masm.movePtr(ImmGCPtr(names.true_), output);
masm.bind(&done);
}
void CodeGenerator::visitIntToString(LIntToString* lir) {
Register input = ToRegister(lir->input());
Register output = ToRegister(lir->output());
using Fn = JSLinearString* (*)(JSContext*,
int);
OutOfLineCode* ool = oolCallVM<Fn, Int32ToString<CanGC>>(
lir, ArgList(input), StoreRegisterTo(output));
masm.lookupStaticIntString(input, output, gen->runtime->staticStrings(),
ool->entry());
masm.bind(ool->rejoin());
}
void CodeGenerator::visitDoubleToString(LDoubleToString* lir) {
FloatRegister input = ToFloatRegister(lir->input());
Register temp = ToRegister(lir->temp0());
Register output = ToRegister(lir->output());
using Fn = JSString* (*)(JSContext*,
double);
OutOfLineCode* ool = oolCallVM<Fn, NumberToString<CanGC>>(
lir, ArgList(input), StoreRegisterTo(output));
// Try double to integer conversion and run integer to string code.
masm.convertDoubleToInt32(input, temp, ool->entry(),
false);
masm.lookupStaticIntString(temp, output, gen->runtime->staticStrings(),
ool->entry());
masm.bind(ool->rejoin());
}
void CodeGenerator::visitValueToString(LValueToString* lir) {
ValueOperand input = ToValue(lir->input());
Register output = ToRegister(lir->output());
using Fn = JSString* (*)(JSContext*, HandleValue);
OutOfLineCode* ool = oolCallVM<Fn, ToStringSlow<CanGC>>(
lir, ArgList(input), StoreRegisterTo(output));
Label done;
Register tag = masm.extractTag(input, output);
const JSAtomState& names = gen->runtime->names();
// String
{
Label notString;
masm.branchTestString(Assembler::NotEqual, tag, ¬String);
masm.unboxString(input, output);
masm.jump(&done);
masm.bind(¬String);
}
// Integer
{
Label notInteger;
masm.branchTestInt32(Assembler::NotEqual, tag, ¬Integer);
Register unboxed = ToTempUnboxRegister(lir->temp0());
unboxed = masm.extractInt32(input, unboxed);
masm.lookupStaticIntString(unboxed, output, gen->runtime->staticStrings(),
ool->entry());
masm.jump(&done);
masm.bind(¬Integer);
}
// Double
{
// Note: no fastpath. Need two extra registers and can only convert doubles
// that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.
masm.branchTestDouble(Assembler::Equal, tag, ool->entry());
}
// Undefined
{
Label notUndefined;
masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined);
masm.movePtr(ImmGCPtr(names.undefined), output);
masm.jump(&done);
masm.bind(¬Undefined);
}
// Null
{
Label notNull;
masm.branchTestNull(Assembler::NotEqual, tag, ¬Null);
masm.movePtr(ImmGCPtr(names.null), output);
masm.jump(&done);
masm.bind(¬Null);
}
// Boolean
{
Label notBoolean, true_;
masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
masm.branchTestBooleanTruthy(
true, input, &true_);
masm.movePtr(ImmGCPtr(names.false_), output);
masm.jump(&done);
masm.bind(&true_);
masm.movePtr(ImmGCPtr(names.true_), output);
masm.jump(&done);
masm.bind(¬Boolean);
}
// Objects/symbols are only possible when |mir->mightHaveSideEffects()|.
if (lir->mir()->mightHaveSideEffects()) {
// Object
--> --------------------
--> maximum size reached
--> --------------------
