| // Copyright 2013 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #if V8_TARGET_ARCH_ARM64 |
| |
| #include "src/api-arguments.h" |
| #include "src/arm64/assembler-arm64-inl.h" |
| #include "src/arm64/macro-assembler-arm64-inl.h" |
| #include "src/bootstrapper.h" |
| #include "src/code-stubs.h" |
| #include "src/counters.h" |
| #include "src/frame-constants.h" |
| #include "src/frames.h" |
| #include "src/ic/ic.h" |
| #include "src/ic/stub-cache.h" |
| #include "src/isolate.h" |
| #include "src/macro-assembler.h" |
| #include "src/objects/api-callbacks.h" |
| #include "src/objects/regexp-match-info.h" |
| #include "src/regexp/jsregexp.h" |
| #include "src/regexp/regexp-macro-assembler.h" |
| #include "src/runtime/runtime.h" |
| |
| #include "src/arm64/code-stubs-arm64.h" // Cannot be the first include. |
| |
| namespace v8 { |
| namespace internal { |
| |
| #define __ ACCESS_MASM(masm) |
| |
| // This is the entry point from C++. 5 arguments are provided in x0-x4. |
| // See use of the JSEntryFunction for example in src/execution.cc. |
| // Input: |
| // x0: code entry. |
| // x1: function. |
| // x2: receiver. |
| // x3: argc. |
| // x4: argv. |
| // Output: |
| // x0: result. |
| void JSEntryStub::Generate(MacroAssembler* masm) { |
| Label invoke, handler_entry, exit; |
| |
| Register code_entry = x0; |
| |
| { |
| NoRootArrayScope no_root_array(masm); |
| |
| // Enable instruction instrumentation. This only works on the simulator, and |
| // will have no effect on the model or real hardware. |
| __ EnableInstrumentation(); |
| |
| __ PushCalleeSavedRegisters(); |
| |
| // Set up the reserved register for 0.0. |
| __ Fmov(fp_zero, 0.0); |
| |
| // Initialize the root array register |
| __ InitializeRootRegister(); |
| } |
| |
| // Build an entry frame (see layout below). |
| StackFrame::Type marker = type(); |
| int64_t bad_frame_pointer = -1L; // Bad frame pointer to fail if it is used. |
| __ Mov(x13, bad_frame_pointer); |
| __ Mov(x12, StackFrame::TypeToMarker(marker)); |
| __ Mov(x11, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, |
| isolate())); |
| __ Ldr(x10, MemOperand(x11)); |
| |
| __ Push(x13, x12, xzr, x10); |
| // Set up fp. |
| __ Sub(fp, sp, EntryFrameConstants::kCallerFPOffset); |
| |
| // Push the JS entry frame marker. Also set js_entry_sp if this is the |
| // outermost JS call. |
| Label non_outermost_js, done; |
| ExternalReference js_entry_sp = |
| ExternalReference::Create(IsolateAddressId::kJSEntrySPAddress, isolate()); |
| __ Mov(x10, js_entry_sp); |
| __ Ldr(x11, MemOperand(x10)); |
| |
| // Select between the inner and outermost frame marker, based on the JS entry |
| // sp. We assert that the inner marker is zero, so we can use xzr to save a |
| // move instruction. |
| DCHECK_EQ(StackFrame::INNER_JSENTRY_FRAME, 0); |
| __ Cmp(x11, 0); // If x11 is zero, this is the outermost frame. |
| __ Csel(x12, xzr, StackFrame::OUTERMOST_JSENTRY_FRAME, ne); |
| __ B(ne, &done); |
| __ Str(fp, MemOperand(x10)); |
| |
| __ Bind(&done); |
| __ Push(x12, padreg); |
| |
| // The frame set up looks like this: |
| // sp[0] : padding. |
| // sp[1] : JS entry frame marker. |
| // sp[2] : C entry FP. |
| // sp[3] : stack frame marker. |
| // sp[4] : stack frame marker. |
| // sp[5] : bad frame pointer 0xFFF...FF <- fp points here. |
| |
| // Jump to a faked try block that does the invoke, with a faked catch |
| // block that sets the pending exception. |
| __ B(&invoke); |
| |
| // Prevent the constant pool from being emitted between the record of the |
| // handler_entry position and the first instruction of the sequence here. |
| // There is no risk because Assembler::Emit() emits the instruction before |
| // checking for constant pool emission, but we do not want to depend on |
| // that. |
| { |
| Assembler::BlockPoolsScope block_pools(masm); |
| __ bind(&handler_entry); |
| handler_offset_ = handler_entry.pos(); |
| // Caught exception: Store result (exception) in the pending exception |
| // field in the JSEnv and return a failure sentinel. Coming in here the |
| // fp will be invalid because the PushTryHandler below sets it to 0 to |
| // signal the existence of the JSEntry frame. |
| __ Mov(x10, Operand(ExternalReference::Create( |
| IsolateAddressId::kPendingExceptionAddress, isolate()))); |
| } |
| __ Str(code_entry, MemOperand(x10)); |
| __ LoadRoot(x0, RootIndex::kException); |
| __ B(&exit); |
| |
| // Invoke: Link this frame into the handler chain. |
| __ Bind(&invoke); |
| |
| // Push new stack handler. |
| static_assert(StackHandlerConstants::kSize == 2 * kPointerSize, |
| "Unexpected offset for StackHandlerConstants::kSize"); |
| static_assert(StackHandlerConstants::kNextOffset == 0 * kPointerSize, |
| "Unexpected offset for StackHandlerConstants::kNextOffset"); |
| |
| // Link the current handler as the next handler. |
| __ Mov(x11, ExternalReference::Create(IsolateAddressId::kHandlerAddress, |
| isolate())); |
| __ Ldr(x10, MemOperand(x11)); |
| __ Push(padreg, x10); |
| |
| // Set this new handler as the current one. |
| { |
| UseScratchRegisterScope temps(masm); |
| Register scratch = temps.AcquireX(); |
| __ Mov(scratch, sp); |
| __ Str(scratch, MemOperand(x11)); |
| } |
| |
| // If an exception not caught by another handler occurs, this handler |
| // returns control to the code after the B(&invoke) above, which |
| // restores all callee-saved registers (including cp and fp) to their |
| // saved values before returning a failure to C. |
| |
| // Invoke the function by calling through the JS entry trampoline builtin. |
| // Notice that we cannot store a reference to the trampoline code directly in |
| // this stub, because runtime stubs are not traversed when doing GC. |
| |
| // Expected registers by Builtins::JSEntryTrampoline |
| // x0: code entry. |
| // x1: function. |
| // x2: receiver. |
| // x3: argc. |
| // x4: argv. |
| __ Call(EntryTrampoline(), RelocInfo::CODE_TARGET); |
| |
| // Pop the stack handler and unlink this frame from the handler chain. |
| static_assert(StackHandlerConstants::kNextOffset == 0 * kPointerSize, |
| "Unexpected offset for StackHandlerConstants::kNextOffset"); |
| __ Pop(x10, padreg); |
| __ Mov(x11, ExternalReference::Create(IsolateAddressId::kHandlerAddress, |
| isolate())); |
| __ Drop(StackHandlerConstants::kSlotCount - 2); |
| __ Str(x10, MemOperand(x11)); |
| |
| __ Bind(&exit); |
| // x0 holds the result. |
| // The stack pointer points to the top of the entry frame pushed on entry from |
| // C++ (at the beginning of this stub): |
| // sp[0] : padding. |
| // sp[1] : JS entry frame marker. |
| // sp[2] : C entry FP. |
| // sp[3] : stack frame marker. |
| // sp[4] : stack frame marker. |
| // sp[5] : bad frame pointer 0xFFF...FF <- fp points here. |
| |
| // Check if the current stack frame is marked as the outermost JS frame. |
| Label non_outermost_js_2; |
| { |
| Register c_entry_fp = x11; |
| __ PeekPair(x10, c_entry_fp, 1 * kPointerSize); |
| __ Cmp(x10, StackFrame::OUTERMOST_JSENTRY_FRAME); |
| __ B(ne, &non_outermost_js_2); |
| __ Mov(x12, js_entry_sp); |
| __ Str(xzr, MemOperand(x12)); |
| __ Bind(&non_outermost_js_2); |
| |
| // Restore the top frame descriptors from the stack. |
| __ Mov(x12, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, |
| isolate())); |
| __ Str(c_entry_fp, MemOperand(x12)); |
| } |
| |
| // Reset the stack to the callee saved registers. |
| static_assert(EntryFrameConstants::kFixedFrameSize % (2 * kPointerSize) == 0, |
| "Size of entry frame is not a multiple of 16 bytes"); |
| __ Drop(EntryFrameConstants::kFixedFrameSize / kPointerSize); |
| // Restore the callee-saved registers and return. |
| __ PopCalleeSavedRegisters(); |
| __ Ret(); |
| } |
| |
| void DirectCEntryStub::Generate(MacroAssembler* masm) { |
| // Put return address on the stack (accessible to GC through exit frame pc). |
| __ Poke(lr, 0); |
| // Call the C++ function. |
| __ Blr(x10); |
| // Return to calling code. |
| __ Peek(lr, 0); |
| __ AssertFPCRState(); |
| __ Ret(); |
| } |
| |
| void DirectCEntryStub::GenerateCall(MacroAssembler* masm, |
| Register target) { |
| // Branch to the stub. |
| __ Mov(x10, target); |
| __ Call(GetCode(), RelocInfo::CODE_TARGET); |
| } |
| |
| // The number of register that CallApiFunctionAndReturn will need to save on |
| // the stack. The space for these registers need to be allocated in the |
| // ExitFrame before calling CallApiFunctionAndReturn. |
| static const int kCallApiFunctionSpillSpace = 4; |
| |
| |
| static int AddressOffset(ExternalReference ref0, ExternalReference ref1) { |
| return static_cast<int>(ref0.address() - ref1.address()); |
| } |
| |
| // Calls an API function. Allocates HandleScope, extracts returned value |
| // from handle and propagates exceptions. |
| // 'stack_space' is the space to be unwound on exit (includes the call JS |
| // arguments space and the additional space allocated for the fast call). |
| // 'spill_offset' is the offset from the stack pointer where |
| // CallApiFunctionAndReturn can spill registers. |
| static void CallApiFunctionAndReturn(MacroAssembler* masm, |
| Register function_address, |
| ExternalReference thunk_ref, |
| int stack_space, int spill_offset, |
| MemOperand return_value_operand) { |
| ASM_LOCATION("CallApiFunctionAndReturn"); |
| Isolate* isolate = masm->isolate(); |
| ExternalReference next_address = |
| ExternalReference::handle_scope_next_address(isolate); |
| const int kNextOffset = 0; |
| const int kLimitOffset = AddressOffset( |
| ExternalReference::handle_scope_limit_address(isolate), next_address); |
| const int kLevelOffset = AddressOffset( |
| ExternalReference::handle_scope_level_address(isolate), next_address); |
| |
| DCHECK(function_address.is(x1) || function_address.is(x2)); |
| |
| Label profiler_disabled; |
| Label end_profiler_check; |
| __ Mov(x10, ExternalReference::is_profiling_address(isolate)); |
| __ Ldrb(w10, MemOperand(x10)); |
| __ Cbz(w10, &profiler_disabled); |
| __ Mov(x3, thunk_ref); |
| __ B(&end_profiler_check); |
| |
| __ Bind(&profiler_disabled); |
| __ Mov(x3, function_address); |
| __ Bind(&end_profiler_check); |
| |
| // Save the callee-save registers we are going to use. |
| // TODO(all): Is this necessary? ARM doesn't do it. |
| STATIC_ASSERT(kCallApiFunctionSpillSpace == 4); |
| __ Poke(x19, (spill_offset + 0) * kXRegSize); |
| __ Poke(x20, (spill_offset + 1) * kXRegSize); |
| __ Poke(x21, (spill_offset + 2) * kXRegSize); |
| __ Poke(x22, (spill_offset + 3) * kXRegSize); |
| |
| // Allocate HandleScope in callee-save registers. |
| // We will need to restore the HandleScope after the call to the API function, |
| // by allocating it in callee-save registers they will be preserved by C code. |
| Register handle_scope_base = x22; |
| Register next_address_reg = x19; |
| Register limit_reg = x20; |
| Register level_reg = w21; |
| |
| __ Mov(handle_scope_base, next_address); |
| __ Ldr(next_address_reg, MemOperand(handle_scope_base, kNextOffset)); |
| __ Ldr(limit_reg, MemOperand(handle_scope_base, kLimitOffset)); |
| __ Ldr(level_reg, MemOperand(handle_scope_base, kLevelOffset)); |
| __ Add(level_reg, level_reg, 1); |
| __ Str(level_reg, MemOperand(handle_scope_base, kLevelOffset)); |
| |
| if (FLAG_log_timer_events) { |
| FrameScope frame(masm, StackFrame::MANUAL); |
| __ PushSafepointRegisters(); |
| __ Mov(x0, ExternalReference::isolate_address(isolate)); |
| __ CallCFunction(ExternalReference::log_enter_external_function(), 1); |
| __ PopSafepointRegisters(); |
| } |
| |
| // Native call returns to the DirectCEntry stub which redirects to the |
| // return address pushed on stack (could have moved after GC). |
| // DirectCEntry stub itself is generated early and never moves. |
| DirectCEntryStub stub(isolate); |
| stub.GenerateCall(masm, x3); |
| |
| if (FLAG_log_timer_events) { |
| FrameScope frame(masm, StackFrame::MANUAL); |
| __ PushSafepointRegisters(); |
| __ Mov(x0, ExternalReference::isolate_address(isolate)); |
| __ CallCFunction(ExternalReference::log_leave_external_function(), 1); |
| __ PopSafepointRegisters(); |
| } |
| |
| Label promote_scheduled_exception; |
| Label delete_allocated_handles; |
| Label leave_exit_frame; |
| Label return_value_loaded; |
| |
| // Load value from ReturnValue. |
| __ Ldr(x0, return_value_operand); |
| __ Bind(&return_value_loaded); |
| // No more valid handles (the result handle was the last one). Restore |
| // previous handle scope. |
| __ Str(next_address_reg, MemOperand(handle_scope_base, kNextOffset)); |
| if (__ emit_debug_code()) { |
| __ Ldr(w1, MemOperand(handle_scope_base, kLevelOffset)); |
| __ Cmp(w1, level_reg); |
| __ Check(eq, AbortReason::kUnexpectedLevelAfterReturnFromApiCall); |
| } |
| __ Sub(level_reg, level_reg, 1); |
| __ Str(level_reg, MemOperand(handle_scope_base, kLevelOffset)); |
| __ Ldr(x1, MemOperand(handle_scope_base, kLimitOffset)); |
| __ Cmp(limit_reg, x1); |
| __ B(ne, &delete_allocated_handles); |
| |
| // Leave the API exit frame. |
| __ Bind(&leave_exit_frame); |
| // Restore callee-saved registers. |
| __ Peek(x19, (spill_offset + 0) * kXRegSize); |
| __ Peek(x20, (spill_offset + 1) * kXRegSize); |
| __ Peek(x21, (spill_offset + 2) * kXRegSize); |
| __ Peek(x22, (spill_offset + 3) * kXRegSize); |
| |
| __ LeaveExitFrame(false, x1, x5); |
| |
| // Check if the function scheduled an exception. |
| __ Mov(x5, ExternalReference::scheduled_exception_address(isolate)); |
| __ Ldr(x5, MemOperand(x5)); |
| __ JumpIfNotRoot(x5, RootIndex::kTheHoleValue, &promote_scheduled_exception); |
| |
| __ DropSlots(stack_space); |
| __ Ret(); |
| |
| // Re-throw by promoting a scheduled exception. |
| __ Bind(&promote_scheduled_exception); |
| __ TailCallRuntime(Runtime::kPromoteScheduledException); |
| |
| // HandleScope limit has changed. Delete allocated extensions. |
| __ Bind(&delete_allocated_handles); |
| __ Str(limit_reg, MemOperand(handle_scope_base, kLimitOffset)); |
| // Save the return value in a callee-save register. |
| Register saved_result = x19; |
| __ Mov(saved_result, x0); |
| __ Mov(x0, ExternalReference::isolate_address(isolate)); |
| __ CallCFunction(ExternalReference::delete_handle_scope_extensions(), 1); |
| __ Mov(x0, saved_result); |
| __ B(&leave_exit_frame); |
| } |
| |
| void CallApiCallbackStub::Generate(MacroAssembler* masm) { |
| // ----------- S t a t e ------------- |
| // -- x4 : call_data |
| // -- x2 : holder |
| // -- x1 : api_function_address |
| // -- cp : context |
| // -- |
| // -- sp[0] : last argument |
| // -- ... |
| // -- sp[(argc - 1) * 8] : first argument |
| // -- sp[argc * 8] : receiver |
| // ----------------------------------- |
| |
| Register call_data = x4; |
| Register holder = x2; |
| Register api_function_address = x1; |
| |
| typedef FunctionCallbackArguments FCA; |
| |
| STATIC_ASSERT(FCA::kArgsLength == 6); |
| STATIC_ASSERT(FCA::kNewTargetIndex == 5); |
| STATIC_ASSERT(FCA::kDataIndex == 4); |
| STATIC_ASSERT(FCA::kReturnValueOffset == 3); |
| STATIC_ASSERT(FCA::kReturnValueDefaultValueIndex == 2); |
| STATIC_ASSERT(FCA::kIsolateIndex == 1); |
| STATIC_ASSERT(FCA::kHolderIndex == 0); |
| |
| Register undef = x7; |
| __ LoadRoot(undef, RootIndex::kUndefinedValue); |
| |
| // Push new target, call data. |
| __ Push(undef, call_data); |
| |
| Register isolate_reg = x5; |
| __ Mov(isolate_reg, ExternalReference::isolate_address(masm->isolate())); |
| |
| // FunctionCallbackArguments: |
| // return value, return value default, isolate, holder. |
| __ Push(undef, undef, isolate_reg, holder); |
| |
| // Prepare arguments. |
| Register args = x6; |
| __ Mov(args, sp); |
| |
| // Allocate the v8::Arguments structure in the arguments' space, since it's |
| // not controlled by GC. |
| const int kApiStackSpace = 3; |
| |
| // Allocate space so that CallApiFunctionAndReturn can store some scratch |
| // registers on the stack. |
| const int kCallApiFunctionSpillSpace = 4; |
| |
| FrameScope frame_scope(masm, StackFrame::MANUAL); |
| __ EnterExitFrame(false, x10, kApiStackSpace + kCallApiFunctionSpillSpace); |
| |
| DCHECK(!AreAliased(x0, api_function_address)); |
| // x0 = FunctionCallbackInfo& |
| // Arguments is after the return address. |
| __ SlotAddress(x0, 1); |
| // FunctionCallbackInfo::implicit_args_ and FunctionCallbackInfo::values_ |
| __ Add(x10, args, Operand((FCA::kArgsLength - 1 + argc()) * kPointerSize)); |
| __ Stp(args, x10, MemOperand(x0, 0 * kPointerSize)); |
| // FunctionCallbackInfo::length_ = argc |
| __ Mov(x10, argc()); |
| __ Str(x10, MemOperand(x0, 2 * kPointerSize)); |
| |
| ExternalReference thunk_ref = ExternalReference::invoke_function_callback(); |
| |
| AllowExternalCallThatCantCauseGC scope(masm); |
| // Stores return the first js argument |
| int return_value_offset = 2 + FCA::kReturnValueOffset; |
| MemOperand return_value_operand(fp, return_value_offset * kPointerSize); |
| // The number of arguments might be odd, but will be padded when calling the |
| // stub. We do not round up stack_space to account for odd argc here, this |
| // will be done in CallApiFunctionAndReturn. |
| const int stack_space = (argc() + 1) + FCA::kArgsLength; |
| |
| // The current frame needs to be aligned. |
| DCHECK_EQ((stack_space - (argc() + 1)) % 2, 0); |
| const int spill_offset = 1 + kApiStackSpace; |
| CallApiFunctionAndReturn(masm, api_function_address, thunk_ref, stack_space, |
| spill_offset, return_value_operand); |
| } |
| |
| |
| void CallApiGetterStub::Generate(MacroAssembler* masm) { |
| STATIC_ASSERT(PropertyCallbackArguments::kShouldThrowOnErrorIndex == 0); |
| STATIC_ASSERT(PropertyCallbackArguments::kHolderIndex == 1); |
| STATIC_ASSERT(PropertyCallbackArguments::kIsolateIndex == 2); |
| STATIC_ASSERT(PropertyCallbackArguments::kReturnValueDefaultValueIndex == 3); |
| STATIC_ASSERT(PropertyCallbackArguments::kReturnValueOffset == 4); |
| STATIC_ASSERT(PropertyCallbackArguments::kDataIndex == 5); |
| STATIC_ASSERT(PropertyCallbackArguments::kThisIndex == 6); |
| STATIC_ASSERT(PropertyCallbackArguments::kArgsLength == 7); |
| |
| Register receiver = ApiGetterDescriptor::ReceiverRegister(); |
| Register holder = ApiGetterDescriptor::HolderRegister(); |
| Register callback = ApiGetterDescriptor::CallbackRegister(); |
| Register data = x4; |
| Register undef = x5; |
| Register isolate_address = x6; |
| Register name = x7; |
| DCHECK(!AreAliased(receiver, holder, callback, data, undef, isolate_address, |
| name)); |
| |
| __ Ldr(data, FieldMemOperand(callback, AccessorInfo::kDataOffset)); |
| __ LoadRoot(undef, RootIndex::kUndefinedValue); |
| __ Mov(isolate_address, ExternalReference::isolate_address(isolate())); |
| __ Ldr(name, FieldMemOperand(callback, AccessorInfo::kNameOffset)); |
| |
| // PropertyCallbackArguments: |
| // receiver, data, return value, return value default, isolate, holder, |
| // should_throw_on_error |
| // These are followed by the property name, which is also pushed below the |
| // exit frame to make the GC aware of it. |
| __ Push(receiver, data, undef, undef, isolate_address, holder, xzr, name); |
| |
| // v8::PropertyCallbackInfo::args_ array and name handle. |
| static const int kStackUnwindSpace = |
| PropertyCallbackArguments::kArgsLength + 1; |
| static_assert(kStackUnwindSpace % 2 == 0, |
| "slots must be a multiple of 2 for stack pointer alignment"); |
| |
| // Load address of v8::PropertyAccessorInfo::args_ array and name handle. |
| __ Mov(x0, sp); // x0 = Handle<Name> |
| __ Add(x1, x0, 1 * kPointerSize); // x1 = v8::PCI::args_ |
| |
| const int kApiStackSpace = 1; |
| |
| // Allocate space so that CallApiFunctionAndReturn can store some scratch |
| // registers on the stack. |
| const int kCallApiFunctionSpillSpace = 4; |
| |
| FrameScope frame_scope(masm, StackFrame::MANUAL); |
| __ EnterExitFrame(false, x10, kApiStackSpace + kCallApiFunctionSpillSpace); |
| |
| // Create v8::PropertyCallbackInfo object on the stack and initialize |
| // it's args_ field. |
| __ Poke(x1, 1 * kPointerSize); |
| __ SlotAddress(x1, 1); |
| // x1 = v8::PropertyCallbackInfo& |
| |
| ExternalReference thunk_ref = |
| ExternalReference::invoke_accessor_getter_callback(); |
| |
| Register api_function_address = x2; |
| Register js_getter = x4; |
| __ Ldr(js_getter, FieldMemOperand(callback, AccessorInfo::kJsGetterOffset)); |
| __ Ldr(api_function_address, |
| FieldMemOperand(js_getter, Foreign::kForeignAddressOffset)); |
| |
| const int spill_offset = 1 + kApiStackSpace; |
| // +3 is to skip prolog, return address and name handle. |
| MemOperand return_value_operand( |
| fp, (PropertyCallbackArguments::kReturnValueOffset + 3) * kPointerSize); |
| CallApiFunctionAndReturn(masm, api_function_address, thunk_ref, |
| kStackUnwindSpace, spill_offset, |
| return_value_operand); |
| } |
| |
| #undef __ |
| |
| } // namespace internal |
| } // namespace v8 |
| |
| #endif // V8_TARGET_ARCH_ARM64 |