// Copyright 2012 the V8 project authors. All rights reserved.

// Use of this source code is governed by a BSD-style license that can be

// found in the LICENSE file.

#include <limits.h> // For LONG_MIN, LONG_MAX.

#if V8_TARGET_ARCH_ARM

#include "src/base/bits.h"

#include "src/base/division-by-constant.h"

#include "src/base/numbers/double.h"

#include "src/base/utils/random-number-generator.h"

#include "src/codegen/assembler-inl.h"

#include "src/codegen/callable.h"

#include "src/codegen/code-factory.h"

#include "src/codegen/external-reference-table.h"

#include "src/codegen/interface-descriptors-inl.h"

#include "src/codegen/macro-assembler.h"

#include "src/codegen/register-configuration.h"

#include "src/debug/debug.h"

#include "src/deoptimizer/deoptimizer.h"

#include "src/execution/frames-inl.h"

#include "src/heap/memory-chunk.h"

#include "src/init/bootstrapper.h"

#include "src/logging/counters.h"

#include "src/objects/objects-inl.h"

#include "src/runtime/runtime.h"

#include "src/snapshot/snapshot.h"

#if V8_ENABLE_WEBASSEMBLY

#include "src/wasm/wasm-code-manager.h"

#endif // V8_ENABLE_WEBASSEMBLY

// Satisfy cpplint check, but don't include platform-specific header. It is

// included recursively via macro-assembler.h.

#if 0

#include "src/codegen/arm/macro-assembler-arm.h"

#endif

namespace v8 {

namespace internal {

int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,

Register exclusion1,

Register exclusion2,

Register exclusion3) const {

int bytes = 0;

RegList exclusions = 0;

if (exclusion1 != no_reg) {

exclusions |= exclusion1.bit();

if (exclusion2 != no_reg) {

exclusions |= exclusion2.bit();

if (exclusion3 != no_reg) {

exclusions |= exclusion3.bit();

}

}

}

RegList list = (kCallerSaved | lr.bit()) & ~exclusions;

bytes += NumRegs(list) * kPointerSize;

if (fp_mode == SaveFPRegsMode::kSave) {

bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;

}

return bytes;

}

int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,

Register exclusion2, Register exclusion3) {

ASM_CODE_COMMENT(this);

int bytes = 0;

RegList exclusions = 0;

if (exclusion1 != no_reg) {

exclusions |= exclusion1.bit();

if (exclusion2 != no_reg) {

exclusions |= exclusion2.bit();

if (exclusion3 != no_reg) {

exclusions |= exclusion3.bit();

}

}

}

RegList list = (kCallerSaved | lr.bit()) & ~exclusions;

stm(db_w, sp, list);

bytes += NumRegs(list) * kPointerSize;

if (fp_mode == SaveFPRegsMode::kSave) {

SaveFPRegs(sp, lr);

bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;

}

return bytes;

}

int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,

Register exclusion2, Register exclusion3) {

ASM_CODE_COMMENT(this);

int bytes = 0;

if (fp_mode == SaveFPRegsMode::kSave) {

RestoreFPRegs(sp, lr);

bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;

}

RegList exclusions = 0;

if (exclusion1 != no_reg) {

exclusions |= exclusion1.bit();

if (exclusion2 != no_reg) {

exclusions |= exclusion2.bit();

if (exclusion3 != no_reg) {

exclusions |= exclusion3.bit();

}

}

}

RegList list = (kCallerSaved | lr.bit()) & ~exclusions;

ldm(ia_w, sp, list);

bytes += NumRegs(list) * kPointerSize;

return bytes;

}

void TurboAssembler::LoadFromConstantsTable(Register destination,

int constant_index) {

DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));

const uint32_t offset =

FixedArray::kHeaderSize + constant_index * kPointerSize - kHeapObjectTag;

LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);

ldr(destination, MemOperand(destination, offset));

}

void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {

ldr(destination, MemOperand(kRootRegister, offset));

}

void TurboAssembler::LoadRootRegisterOffset(Register destination,

intptr_t offset) {

if (offset == 0) {

Move(destination, kRootRegister);

} else {

add(destination, kRootRegister, Operand(offset));

}

}

void TurboAssembler::Jump(Register target, Condition cond) { bx(target, cond); }

void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,

Condition cond) {

mov(pc, Operand(target, rmode), LeaveCC, cond);

}

void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,

Condition cond) {

DCHECK(!RelocInfo::IsCodeTarget(rmode));

Jump(static_cast<intptr_t>(target), rmode, cond);

}

void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,

Condition cond) {

DCHECK(RelocInfo::IsCodeTarget(rmode));

DCHECK_IMPLIES(options().isolate_independent_code,

Builtins::IsIsolateIndependentBuiltin(*code));

DCHECK_IMPLIES(options().use_pc_relative_calls_and_jumps,

Builtins::IsIsolateIndependentBuiltin(*code));

Builtin builtin = Builtin::kNoBuiltinId;

bool target_is_builtin =

isolate()->builtins()->IsBuiltinHandle(code, &builtin);

if (options().use_pc_relative_calls_and_jumps && target_is_builtin) {

int32_t code_target_index = AddCodeTarget(code);

b(code_target_index * kInstrSize, cond, RelocInfo::RELATIVE_CODE_TARGET);

return;

} else if (root_array_available_ && options().isolate_independent_code) {

// This branch is taken only for specific cctests, where we force isolate

// creation at runtime. At this point, Code space isn't restricted to a

// size s.t. pc-relative calls may be used.

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

int offset = IsolateData::BuiltinEntrySlotOffset(code->builtin_id());

ldr(scratch, MemOperand(kRootRegister, offset));

Jump(scratch, cond);

return;

} else if (options().inline_offheap_trampolines && target_is_builtin) {

// Inline the trampoline.

RecordCommentForOffHeapTrampoline(builtin);

// Use ip directly instead of using UseScratchRegisterScope, as we do not

// preserve scratch registers across calls.

mov(ip, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));

Jump(ip, cond);

return;

}

// 'code' is always generated ARM code, never THUMB code

Jump(static_cast<intptr_t>(code.address()), rmode, cond);

}

void TurboAssembler::Jump(const ExternalReference& reference) {

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

Move(scratch, reference);

Jump(scratch);

}

void TurboAssembler::Call(Register target, Condition cond) {

// Block constant pool for the call instruction sequence.

BlockConstPoolScope block_const_pool(this);

blx(target, cond);

}

void TurboAssembler::Call(Address target, RelocInfo::Mode rmode, Condition cond,

TargetAddressStorageMode mode,

bool check_constant_pool) {

// Check if we have to emit the constant pool before we block it.

if (check_constant_pool) MaybeCheckConstPool();

// Block constant pool for the call instruction sequence.

BlockConstPoolScope block_const_pool(this);

bool old_predictable_code_size = predictable_code_size();

if (mode == NEVER_INLINE_TARGET_ADDRESS) {

set_predictable_code_size(true);

}

// Use ip directly instead of using UseScratchRegisterScope, as we do not

// preserve scratch registers across calls.

// Call sequence on V7 or later may be :

// movw ip, #... @ call address low 16

// movt ip, #... @ call address high 16

// blx ip

// @ return address

// Or for pre-V7 or values that may be back-patched

// to avoid ICache flushes:

// ldr ip, [pc, #...] @ call address

// blx ip

// @ return address

mov(ip, Operand(target, rmode));

blx(ip, cond);

if (mode == NEVER_INLINE_TARGET_ADDRESS) {

set_predictable_code_size(old_predictable_code_size);

}

}

void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,

Condition cond, TargetAddressStorageMode mode,

bool check_constant_pool) {

DCHECK(RelocInfo::IsCodeTarget(rmode));

DCHECK_IMPLIES(options().isolate_independent_code,

Builtins::IsIsolateIndependentBuiltin(*code));

DCHECK_IMPLIES(options().use_pc_relative_calls_and_jumps,

Builtins::IsIsolateIndependentBuiltin(*code));

Builtin builtin = Builtin::kNoBuiltinId;

bool target_is_builtin =

isolate()->builtins()->IsBuiltinHandle(code, &builtin);

if (target_is_builtin && options().use_pc_relative_calls_and_jumps) {

int32_t code_target_index = AddCodeTarget(code);

bl(code_target_index * kInstrSize, cond, RelocInfo::RELATIVE_CODE_TARGET);

return;

} else if (root_array_available_ && options().isolate_independent_code) {

// This branch is taken only for specific cctests, where we force isolate

// creation at runtime. At this point, Code space isn't restricted to a

// size s.t. pc-relative calls may be used.

int offset = IsolateData::BuiltinEntrySlotOffset(code->builtin_id());

ldr(ip, MemOperand(kRootRegister, offset));

Call(ip, cond);

return;

} else if (target_is_builtin && options().inline_offheap_trampolines) {

// Inline the trampoline.

CallBuiltin(builtin);

return;

}

// 'code' is always generated ARM code, never THUMB code

DCHECK(code->IsExecutable());

Call(code.address(), rmode, cond, mode);

}

void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {

ASM_CODE_COMMENT(this);

STATIC_ASSERT(kSystemPointerSize == 4);

STATIC_ASSERT(kSmiShiftSize == 0);

STATIC_ASSERT(kSmiTagSize == 1);

STATIC_ASSERT(kSmiTag == 0);

// The builtin_index register contains the builtin index as a Smi.

// Untagging is folded into the indexing operand below.

mov(builtin_index,

Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiTagSize));

add(builtin_index, builtin_index,

Operand(IsolateData::builtin_entry_table_offset()));

ldr(builtin_index, MemOperand(kRootRegister, builtin_index));

}

void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {

LoadEntryFromBuiltinIndex(builtin_index);

Call(builtin_index);

}

void TurboAssembler::LoadEntryFromBuiltin(Builtin builtin,

Register destination) {

ASM_CODE_COMMENT(this);

ldr(destination, EntryFromBuiltinAsOperand(builtin));

}

MemOperand TurboAssembler::EntryFromBuiltinAsOperand(Builtin builtin) {

ASM_CODE_COMMENT(this);

DCHECK(root_array_available());

return MemOperand(kRootRegister,

IsolateData::BuiltinEntrySlotOffset(builtin));

}

void TurboAssembler::CallBuiltin(Builtin builtin, Condition cond) {

ASM_CODE_COMMENT_STRING(this, CommentForOffHeapTrampoline("call", builtin));

DCHECK(Builtins::IsBuiltinId(builtin));

// Use ip directly instead of using UseScratchRegisterScope, as we do not

// preserve scratch registers across calls.

mov(ip, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));

Call(ip, cond);

}

void TurboAssembler::LoadCodeObjectEntry(Register destination,

Register code_object) {

ASM_CODE_COMMENT(this);

// Code objects are called differently depending on whether we are generating

// builtin code (which will later be embedded into the binary) or compiling

// user JS code at runtime.

// * Builtin code runs in --jitless mode and thus must not call into on-heap

// Code targets. Instead, we dispatch through the builtins entry table.

// * Codegen at runtime does not have this restriction and we can use the

// shorter, branchless instruction sequence. The assumption here is that

// targets are usually generated code and not builtin Code objects.

if (options().isolate_independent_code) {

DCHECK(root_array_available());

Label if_code_is_off_heap, out;

{

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

DCHECK(!AreAliased(destination, scratch));

DCHECK(!AreAliased(code_object, scratch));

// Check whether the Code object is an off-heap trampoline. If so, call

// its (off-heap) entry point directly without going through the (on-heap)

// trampoline. Otherwise, just call the Code object as always.

ldr(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));

tst(scratch, Operand(Code::IsOffHeapTrampoline::kMask));

b(ne, &if_code_is_off_heap);

// Not an off-heap trampoline, the entry point is at

// Code::raw_instruction_start().

add(destination, code_object,

Operand(Code::kHeaderSize - kHeapObjectTag));

jmp(&out);

// An off-heap trampoline, the entry point is loaded from the builtin

// entry table.

bind(&if_code_is_off_heap);

ldr(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));

lsl(destination, scratch, Operand(kSystemPointerSizeLog2));

}

add(destination, destination, kRootRegister);

ldr(destination,

MemOperand(destination, IsolateData::builtin_entry_table_offset()));

bind(&out);

} else {

add(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));

}

}

void TurboAssembler::CallCodeObject(Register code_object) {

ASM_CODE_COMMENT(this);

LoadCodeObjectEntry(code_object, code_object);

Call(code_object);

}

void TurboAssembler::JumpCodeObject(Register code_object, JumpMode jump_mode) {

ASM_CODE_COMMENT(this);

DCHECK_EQ(JumpMode::kJump, jump_mode);

LoadCodeObjectEntry(code_object, code_object);

Jump(code_object);

}

void TurboAssembler::StoreReturnAddressAndCall(Register target) {

ASM_CODE_COMMENT(this);

// This generates the final instruction sequence for calls to C functions

// once an exit frame has been constructed.

//

// Note that this assumes the caller code (i.e. the Code object currently

// being generated) is immovable or that the callee function cannot trigger

// GC, since the callee function will return to it.

// Compute the return address in lr to return to after the jump below. The pc

// is already at '+ 8' from the current instruction; but return is after three

// instructions, so add another 4 to pc to get the return address.

Assembler::BlockConstPoolScope block_const_pool(this);

add(lr, pc, Operand(4));

str(lr, MemOperand(sp));

Call(target);

}

void TurboAssembler::Ret(Condition cond) { bx(lr, cond); }

void TurboAssembler::Drop(int count, Condition cond) {

if (count > 0) {

add(sp, sp, Operand(count * kPointerSize), LeaveCC, cond);

}

}

void TurboAssembler::Drop(Register count, Condition cond) {

add(sp, sp, Operand(count, LSL, kPointerSizeLog2), LeaveCC, cond);

}

void TurboAssembler::Ret(int drop, Condition cond) {

Drop(drop, cond);

Ret(cond);

}

void TurboAssembler::Call(Label* target) { bl(target); }

void TurboAssembler::Push(Handle<HeapObject> handle) {

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

mov(scratch, Operand(handle));

push(scratch);

}

void TurboAssembler::Push(Smi smi) {

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

mov(scratch, Operand(smi));

push(scratch);

}

void TurboAssembler::PushArray(Register array, Register size, Register scratch,

PushArrayOrder order) {

ASM_CODE_COMMENT(this);

UseScratchRegisterScope temps(this);

Register counter = scratch;

Register tmp = temps.Acquire();

DCHECK(!AreAliased(array, size, counter, tmp));

Label loop, entry;

if (order == PushArrayOrder::kReverse) {

mov(counter, Operand(0));

b(&entry);

bind(&loop);

ldr(tmp, MemOperand(array, counter, LSL, kSystemPointerSizeLog2));

push(tmp);

add(counter, counter, Operand(1));

bind(&entry);

cmp(counter, size);

b(lt, &loop);

} else {

mov(counter, size);

b(&entry);

bind(&loop);

ldr(tmp, MemOperand(array, counter, LSL, kSystemPointerSizeLog2));

push(tmp);

bind(&entry);

sub(counter, counter, Operand(1), SetCC);

b(ge, &loop);

}

}

void TurboAssembler::Move(Register dst, Smi smi) { mov(dst, Operand(smi)); }

void TurboAssembler::Move(Register dst, Handle<HeapObject> value) {

// TODO(jgruber,v8:8887): Also consider a root-relative load when generating

// non-isolate-independent code. In many cases it might be cheaper than

// embedding the relocatable value.

if (root_array_available_ && options().isolate_independent_code) {

IndirectLoadConstant(dst, value);

return;

}

mov(dst, Operand(value));

}

void TurboAssembler::Move(Register dst, ExternalReference reference) {

// TODO(jgruber,v8:8887): Also consider a root-relative load when generating

// non-isolate-independent code. In many cases it might be cheaper than

// embedding the relocatable value.

if (root_array_available_ && options().isolate_independent_code) {

IndirectLoadExternalReference(dst, reference);

return;

}

mov(dst, Operand(reference));

}

void TurboAssembler::Move(Register dst, Register src, Condition cond) {

if (dst != src) {

mov(dst, src, LeaveCC, cond);

}

}

void TurboAssembler::Move(SwVfpRegister dst, SwVfpRegister src,

Condition cond) {

if (dst != src) {

vmov(dst, src, cond);

}

}

void TurboAssembler::Move(DwVfpRegister dst, DwVfpRegister src,

Condition cond) {

if (dst != src) {

vmov(dst, src, cond);

}

}

void TurboAssembler::Move(QwNeonRegister dst, QwNeonRegister src) {

if (dst != src) {

vmov(dst, src);

}

}

void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,

Register src1) {

DCHECK_NE(dst0, dst1);

if (dst0 != src1) {

Move(dst0, src0);

Move(dst1, src1);

} else if (dst1 != src0) {

// Swap the order of the moves to resolve the overlap.

Move(dst1, src1);

Move(dst0, src0);

} else {

// Worse case scenario, this is a swap.

Swap(dst0, src0);

}

}

void TurboAssembler::Swap(Register srcdst0, Register srcdst1) {

DCHECK(srcdst0 != srcdst1);

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

mov(scratch, srcdst0);

mov(srcdst0, srcdst1);

mov(srcdst1, scratch);

}

void TurboAssembler::Swap(DwVfpRegister srcdst0, DwVfpRegister srcdst1) {

DCHECK(srcdst0 != srcdst1);

DCHECK(VfpRegisterIsAvailable(srcdst0));

DCHECK(VfpRegisterIsAvailable(srcdst1));

if (CpuFeatures::IsSupported(NEON)) {

vswp(srcdst0, srcdst1);

} else {

UseScratchRegisterScope temps(this);

DwVfpRegister scratch = temps.AcquireD();

vmov(scratch, srcdst0);

vmov(srcdst0, srcdst1);

vmov(srcdst1, scratch);

}

}

void TurboAssembler::Swap(QwNeonRegister srcdst0, QwNeonRegister srcdst1) {

DCHECK(srcdst0 != srcdst1);

vswp(srcdst0, srcdst1);

}

void MacroAssembler::Mls(Register dst, Register src1, Register src2,

Register srcA, Condition cond) {

if (CpuFeatures::IsSupported(ARMv7)) {

CpuFeatureScope scope(this, ARMv7);

mls(dst, src1, src2, srcA, cond);

} else {

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

DCHECK(srcA != scratch);

mul(scratch, src1, src2, LeaveCC, cond);

sub(dst, srcA, scratch, LeaveCC, cond);

}

}

void MacroAssembler::And(Register dst, Register src1, const Operand& src2,

Condition cond) {

if (!src2.IsRegister() && !src2.MustOutputRelocInfo(this) &&

src2.immediate() == 0) {

mov(dst, Operand::Zero(), LeaveCC, cond);

} else if (!(src2.InstructionsRequired(this) == 1) &&

!src2.MustOutputRelocInfo(this) &&

CpuFeatures::IsSupported(ARMv7) &&

base::bits::IsPowerOfTwo(src2.immediate() + 1)) {

CpuFeatureScope scope(this, ARMv7);

ubfx(dst, src1, 0,

base::bits::WhichPowerOfTwo(static_cast<uint32_t>(src2.immediate()) +

1),

cond);

} else {

and_(dst, src1, src2, LeaveCC, cond);

}

}

void MacroAssembler::Ubfx(Register dst, Register src1, int lsb, int width,

Condition cond) {

DCHECK_LT(lsb, 32);

if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {

int mask = (1u << (width + lsb)) - 1u - ((1u << lsb) - 1u);

and_(dst, src1, Operand(mask), LeaveCC, cond);

if (lsb != 0) {

mov(dst, Operand(dst, LSR, lsb), LeaveCC, cond);

}

} else {

CpuFeatureScope scope(this, ARMv7);

ubfx(dst, src1, lsb, width, cond);

}

}

void MacroAssembler::Sbfx(Register dst, Register src1, int lsb, int width,

Condition cond) {

DCHECK_LT(lsb, 32);

if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {

int mask = (1 << (width + lsb)) - 1 - ((1 << lsb) - 1);

and_(dst, src1, Operand(mask), LeaveCC, cond);

int shift_up = 32 - lsb - width;

int shift_down = lsb + shift_up;

if (shift_up != 0) {

mov(dst, Operand(dst, LSL, shift_up), LeaveCC, cond);

}

if (shift_down != 0) {

mov(dst, Operand(dst, ASR, shift_down), LeaveCC, cond);

}

} else {

CpuFeatureScope scope(this, ARMv7);

sbfx(dst, src1, lsb, width, cond);

}

}

void TurboAssembler::Bfc(Register dst, Register src, int lsb, int width,

Condition cond) {

DCHECK_LT(lsb, 32);

if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {

int mask = (1 << (width + lsb)) - 1 - ((1 << lsb) - 1);

bic(dst, src, Operand(mask));

} else {

CpuFeatureScope scope(this, ARMv7);

Move(dst, src, cond);

bfc(dst, lsb, width, cond);

}

}

void TurboAssembler::LoadRoot(Register destination, RootIndex index,

Condition cond) {

ldr(destination,

MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)), cond);

}

void MacroAssembler::RecordWriteField(Register object, int offset,

Register value,

LinkRegisterStatus lr_status,

SaveFPRegsMode save_fp,

RememberedSetAction remembered_set_action,

SmiCheck smi_check) {

ASM_CODE_COMMENT(this);

// First, check if a write barrier is even needed. The tests below

// catch stores of Smis.

Label done;

// Skip barrier if writing a smi.

if (smi_check == SmiCheck::kInline) {

JumpIfSmi(value, &done);

}

// Although the object register is tagged, the offset is relative to the start

// of the object, so so offset must be a multiple of kPointerSize.

DCHECK(IsAligned(offset, kPointerSize));

if (FLAG_debug_code) {

ASM_CODE_COMMENT_STRING(this, "Verify slot_address");

Label ok;

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

DCHECK(!AreAliased(object, value, scratch));

add(scratch, object, Operand(offset - kHeapObjectTag));

tst(scratch, Operand(kPointerSize - 1));

b(eq, &ok);

stop();

bind(&ok);

}

RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,

save_fp, remembered_set_action, SmiCheck::kOmit);

bind(&done);

}

void TurboAssembler::MaybeSaveRegisters(RegList registers) {

if (registers == 0) return;

ASM_CODE_COMMENT(this);

RegList regs = 0;

for (int i = 0; i < Register::kNumRegisters; ++i) {

if ((registers >> i) & 1u) {

regs |= Register::from_code(i).bit();

}

}

stm(db_w, sp, regs);

}

void TurboAssembler::MaybeRestoreRegisters(RegList registers) {

if (registers == 0) return;

ASM_CODE_COMMENT(this);

RegList regs = 0;

for (int i = 0; i < Register::kNumRegisters; ++i) {

if ((registers >> i) & 1u) {

regs |= Register::from_code(i).bit();

}

}

ldm(ia_w, sp, regs);

}

void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,

SaveFPRegsMode fp_mode) {

ASM_CODE_COMMENT(this);

RegList registers = WriteBarrierDescriptor::ComputeSavedRegisters(object);

MaybeSaveRegisters(registers);

Register object_parameter = WriteBarrierDescriptor::ObjectRegister();

Register slot_address_parameter =

WriteBarrierDescriptor::SlotAddressRegister();

MoveObjectAndSlot(object_parameter, slot_address_parameter, object, offset);

Call(isolate()->builtins()->code_handle(

Builtins::GetEphemeronKeyBarrierStub(fp_mode)),

RelocInfo::CODE_TARGET);

MaybeRestoreRegisters(registers);

}

void TurboAssembler::CallRecordWriteStubSaveRegisters(

Register object, Operand offset, RememberedSetAction remembered_set_action,

SaveFPRegsMode fp_mode, StubCallMode mode) {

ASM_CODE_COMMENT(this);

RegList registers = WriteBarrierDescriptor::ComputeSavedRegisters(object);

MaybeSaveRegisters(registers);

Register object_parameter = WriteBarrierDescriptor::ObjectRegister();

Register slot_address_parameter =

WriteBarrierDescriptor::SlotAddressRegister();

MoveObjectAndSlot(object_parameter, slot_address_parameter, object, offset);

CallRecordWriteStub(object_parameter, slot_address_parameter,

remembered_set_action, fp_mode, mode);

MaybeRestoreRegisters(registers);

}

void TurboAssembler::CallRecordWriteStub(

Register object, Register slot_address,

RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,

StubCallMode mode) {

ASM_CODE_COMMENT(this);

DCHECK_EQ(WriteBarrierDescriptor::ObjectRegister(), object);

DCHECK_EQ(WriteBarrierDescriptor::SlotAddressRegister(), slot_address);

#if V8_ENABLE_WEBASSEMBLY

if (mode == StubCallMode::kCallWasmRuntimeStub) {

auto wasm_target =

wasm::WasmCode::GetRecordWriteStub(remembered_set_action, fp_mode);

Call(wasm_target, RelocInfo::WASM_STUB_CALL);

#else

if (false) {

#endif

} else {

Builtin builtin =

Builtins::GetRecordWriteStub(remembered_set_action, fp_mode);

if (options().inline_offheap_trampolines) {

CallBuiltin(builtin);

} else {

Handle<Code> code_target = isolate()->builtins()->code_handle(builtin);

Call(code_target, RelocInfo::CODE_TARGET);

}

}

}

void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,

Register object, Operand offset) {

DCHECK_NE(dst_object, dst_slot);

DCHECK(offset.IsRegister() || offset.IsImmediate());

// If `offset` is a register, it cannot overlap with `object`.

DCHECK_IMPLIES(offset.IsRegister(), offset.rm() != object);

// If the slot register does not overlap with the object register, we can

// overwrite it.

if (dst_slot != object) {

add(dst_slot, object, offset);

Move(dst_object, object);

return;

}

DCHECK_EQ(dst_slot, object);

// If the destination object register does not overlap with the offset

// register, we can overwrite it.

if (!offset.IsRegister() || (offset.rm() != dst_object)) {

Move(dst_object, dst_slot);

add(dst_slot, dst_slot, offset);

return;

}

DCHECK_EQ(dst_object, offset.rm());

// We only have `dst_slot` and `dst_object` left as distinct registers so we

// have to swap them. We write this as a add+sub sequence to avoid using a

// scratch register.

add(dst_slot, dst_slot, dst_object);

sub(dst_object, dst_slot, dst_object);

}

// The register 'object' contains a heap object pointer. The heap object tag is

// shifted away. A scratch register also needs to be available.

void MacroAssembler::RecordWrite(Register object, Operand offset,

Register value, LinkRegisterStatus lr_status,

SaveFPRegsMode fp_mode,

RememberedSetAction remembered_set_action,

SmiCheck smi_check) {

DCHECK(!AreAliased(object, value));

if (FLAG_debug_code) {

ASM_CODE_COMMENT_STRING(this, "Verify slot_address");

UseScratchRegisterScope temps(this);

Register scratch = temps.Acquire();

DCHECK(!AreAliased(object, value, scratch));

add(scratch, object, offset);

ldr(scratch, MemOperand(scratch));

cmp(scratch, value);

Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);

}

if ((remembered_set_action == RememberedSetAction::kOmit &&

!FLAG_incremental_marking) ||

FLAG_disable_write_barriers) {

return;

}

// First, check if a write barrier is even needed. The tests below

// catch stores of smis and stores into the young generation.

Label done;

if (smi_check == SmiCheck::kInline) {

JumpIfSmi(value, &done);

}

CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, eq,

&done);

CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, eq,

&done);

// Record the actual write.

if (lr_status == kLRHasNotBeenSaved) {

push(lr);

}

Register slot_address = WriteBarrierDescriptor::SlotAddressRegister();

DCHECK(!AreAliased(object, value, slot_address));

DCHECK(!offset.IsRegister());

add(slot_address, object, offset);

CallRecordWriteStub(object, slot_address, remembered_set_action, fp_mode);

if (lr_status == kLRHasNotBeenSaved) {

pop(lr);

}

if (FLAG_debug_code) Move(slot_address, Operand(kZapValue));

bind(&done);

}

void TurboAssembler::PushCommonFrame(Register marker_reg) {

ASM_CODE_COMMENT(this);

if (marker_reg.is_valid()) {

if (marker_reg.code() > fp.code()) {

stm(db_w, sp, fp.bit() | lr.bit());

mov(fp, Operand(sp));

Push(marker_reg);

} else {

stm(db_w, sp, marker_reg.bit() | fp.bit() | lr.bit());

add(fp, sp, Operand(kPointerSize));

}

} else {

stm(db_w, sp, fp.bit() | lr.bit());

mov(fp, sp);

}

}

void TurboAssembler::PushStandardFrame(Register function_reg) {

ASM_CODE_COMMENT(this);

DCHECK(!function_reg.is_valid() || function_reg.code() < cp.code());

stm(db_w, sp,

(function_reg.is_valid() ? function_reg.bit() : 0) | cp.bit() | fp.bit() |

lr.bit());

int offset = -StandardFrameConstants::kContextOffset;

offset += function_reg.is_valid() ? kPointerSize : 0;

add(fp, sp, Operand(offset));

Push(kJavaScriptCallArgCountRegister);

}

void TurboAssembler::VFPCanonicalizeNaN(const DwVfpRegister dst,

const DwVfpRegister src,

const Condition cond) {

// Subtracting 0.0 preserves all inputs except for signalling NaNs, which

// become quiet NaNs. We use vsub rather than vadd because vsub preserves -0.0

// inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.

vsub(dst, src, kDoubleRegZero, cond);

}

void TurboAssembler::VFPCompareAndSetFlags(const SwVfpRegister src1,

const SwVfpRegister src2,

const Condition cond) {

// Compare and move FPSCR flags to the normal condition flags.

VFPCompareAndLoadFlags(src1, src2, pc, cond);

}

void TurboAssembler::VFPCompareAndSetFlags(const SwVfpRegister src1,

const float src2,

const Condition cond) {

// Compare and move FPSCR flags to the normal condition flags.

VFPCompareAndLoadFlags(src1, src2, pc, cond);

}

void TurboAssembler::VFPCompareAndSetFlags(const DwVfpRegister src1,

const DwVfpRegister src2,

const Condition cond) {

// Compare and move FPSCR flags to the normal condition flags.

VFPCompareAndLoadFlags(src1, src2, pc, cond);

}

void TurboAssembler::VFPCompareAndSetFlags(const DwVfpRegister src1,

const double src2,

const Condition cond) {

// Compare and move FPSCR flags to the normal condition flags.

VFPCompareAndLoadFlags(src1, src2, pc, cond);

}

void TurboAssembler::VFPCompareAndLoadFlags(const SwVfpRegister src1,

const SwVfpRegister src2,

const Register fpscr_flags,

const Condition cond) {

// Compare and load FPSCR.

vcmp(src1, src2, cond);

vmrs(fpscr_flags, cond);

}

void TurboAssembler::VFPCompareAndLoadFlags(const SwVfpRegister src1,

const float src2,

const Register fpscr_flags,

const Condition cond) {

// Compare and load FPSCR.

vcmp(src1, src2, cond);

vmrs(fpscr_flags, cond);

}

void TurboAssembler::VFPCompareAndLoadFlags(const DwVfpRegister src1,

const DwVfpRegister src2,

const Register fpscr_flags,

const Condition cond) {

// Compare and load FPSCR.

vcmp(src1, src2, cond);

vmrs(fpscr_flags, cond);

}

void TurboAssembler::VFPCompareAndLoadFlags(const DwVfpRegister src1,

const double src2,

const Register fpscr_flags,

const Condition cond) {

// Compare and load FPSCR.

vcmp(src1, src2, cond);

vmrs(fpscr_flags, cond);

}

void TurboAssembler::VmovHigh(Register dst, DwVfpRegister src) {

if (src.code() < 16) {

const LowDwVfpRegister loc = LowDwVfpRegister::from_code(src.code());

vmov(dst, loc.high());

} else {

vmov(NeonS32, dst, src, 1);

}

}

void TurboAssembler::VmovHigh(DwVfpRegister dst, Register src) {

if (dst.code() < 16) {

const LowDwVfpRegister loc = LowDwVfpRegister::from_code(dst.code());

vmov(loc.high(), src);

} else {

vmov(NeonS32, dst, 1, src);

}

}

void TurboAssembler::VmovLow(Register dst, DwVfpRegister src) {

if (src.code() < 16) {

const LowDwVfpRegister loc = LowDwVfpRegister::from_code(src.code());

vmov(dst, loc.low());

} else {

vmov(NeonS32, dst, src, 0);