//-------------------------------------------------------------------------------------------------------

// Copyright (C) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.

//-------------------------------------------------------------------------------------------------------

#include "BackEnd.h"

#include "Base\ScriptContextProfiler.h"

#if DBG

Js::JavascriptMethod checkCodeGenThunk;

#endif

#ifdef ENABLE_PREJIT

#define IS_PREJIT_ON() (Js::Configuration::Global.flags.Prejit)

#else

#define IS_PREJIT_ON() (DEFAULT_CONFIG_Prejit)

#endif

#define ASSERT_THREAD() AssertMsg(mainThreadId == GetCurrentThreadContextId(), \

"Cannot use this member of native code generator from thread other than the creating context's current thread")

NativeCodeGenerator::NativeCodeGenerator(Js::ScriptContext * scriptContext)

: JsUtil::WaitableJobManager(scriptContext->GetThreadContext()->GetJobProcessor()),

scriptContext(scriptContext),

pendingCodeGenWorkItems(0),

queuedFullJitWorkItemCount(0),

foregroundAllocators(nullptr),

backgroundAllocators(nullptr),

byteCodeSizeGenerated(0),

isClosed(false),

isOptimizedForManyInstances(scriptContext->GetThreadContext()->IsOptimizedForManyInstances()),

SetNativeEntryPoint(Js::FunctionBody::DefaultSetNativeEntryPoint),

freeLoopBodyManager(scriptContext->GetThreadContext()->GetJobProcessor()),

hasUpdatedQForDebugMode(false)

#ifdef PROFILE_EXEC

, foregroundCodeGenProfiler(nullptr)

, backgroundCodeGenProfiler(nullptr)

#endif

{

freeLoopBodyManager.SetNativeCodeGen(this);

#if DBG_DUMP

if (Js::Configuration::Global.flags.IsEnabled(Js::AsmDumpModeFlag)

&& (Js::Configuration::Global.flags.AsmDumpMode != nullptr))

{

bool fileOpened = false;

fileOpened = (0 == _wfopen_s(&this->asmFile, Js::Configuration::Global.flags.AsmDumpMode, L"wt"));

if (!fileOpened)

{

size_t len = wcslen(Js::Configuration::Global.flags.AsmDumpMode);

if (len < _MAX_PATH - 5)

{

wchar_t filename[_MAX_PATH];

wcscpy_s(filename, _MAX_PATH, Js::Configuration::Global.flags.AsmDumpMode);

wchar_t * number = filename + len;

for (int i = 0; i < 1000; i++)

{

_itow_s(i, number, 5, 10);

fileOpened = (0 == _wfopen_s(&this->asmFile, filename, L"wt"));

if (fileOpened)

{

break;

}

}

}

if (!fileOpened)

{

this->asmFile = nullptr;

AssertMsg(0, "Could not open file for AsmDump. The output will goto standard console");

}

}

}

else

{

this->asmFile = nullptr;

}

#endif

#if DBG

this->mainThreadId = GetCurrentThreadContextId();

#endif

Processor()->AddManager(this);

this->freeLoopBodyManager.SetAutoClose(false);

}

NativeCodeGenerator::~NativeCodeGenerator()

{

Assert(this->IsClosed());

#ifdef PROFILE_EXEC

if (this->foregroundCodeGenProfiler != nullptr)

{

this->foregroundCodeGenProfiler->Release();

}

#endif

if(this->foregroundAllocators != nullptr)

{

HeapDelete(this->foregroundAllocators);

}

if (this->backgroundAllocators)

{

#if DBG

// PageAllocator is thread agile. This destructor can be called from background GC thread.

// We have already removed this manager from the job queue and hence its fine to set the threadId to -1.

// We can't DissociatePageAllocator here as its allocated ui thread.

//this->Processor()->DissociatePageAllocator(allocator->GetPageAllocator());

this->backgroundAllocators->emitBufferManager.GetHeapPageAllocator()->ClearConcurrentThreadId();

this->backgroundAllocators->emitBufferManager.GetPreReservedHeapPageAllocator()->ClearConcurrentThreadId();

this->backgroundAllocators->GetPageAllocator()->ClearConcurrentThreadId();

#endif

// The native code generator may be deleted after Close was called on the job processor. In that case, the

// background thread is no longer running, so clean things up in the foreground.

HeapDelete(this->backgroundAllocators);

}

#ifdef PROFILE_EXEC

if (Js::Configuration::Global.flags.IsEnabled(Js::ProfileFlag))

{

while (this->backgroundCodeGenProfiler)

{

Js::ScriptContextProfiler *codegenProfiler = this->backgroundCodeGenProfiler;

this->backgroundCodeGenProfiler = this->backgroundCodeGenProfiler->next;

codegenProfiler->Release();

}

}

else

{

Assert(this->backgroundCodeGenProfiler == nullptr);

}

#endif

}

void NativeCodeGenerator::Close()

{

Assert(!this->IsClosed());

// Close FreeLoopBodyJobManager first, as it depends on NativeCodeGenerator to be open before it's removed

this->freeLoopBodyManager.Close();

// Remove only if it is not updated in the debug mode (and which goes to interpreter mode).

if (!hasUpdatedQForDebugMode || Js::Configuration::Global.EnableJitInDebugMode())

{

Processor()->RemoveManager(this);

}

this->isClosed = true;

Assert(!queuedFullJitWorkItems.Head());

Assert(queuedFullJitWorkItemCount == 0);

for(JsUtil::Job *job = workItems.Head(); job;)

{

JsUtil::Job *const next = job->Next();

JobProcessed(job, /*succeeded*/ false);

job = next;

}

workItems.Clear();

// Only decommit here instead of releasing the memory, so we retain control over these addresses

// Mitigate against the case the entry point is called after the script site is closed

if (this->backgroundAllocators)

{

this->backgroundAllocators->emitBufferManager.Decommit();

}

if (this->foregroundAllocators)

{

this->foregroundAllocators->emitBufferManager.Decommit();

}

#if DBG_DUMP

if (this->asmFile != nullptr)

{

if(0 != fclose(this->asmFile))

{

AssertMsg(0, "Could not close file for AsmDump. You may ignore this warning.");

}

}

#endif

}

#if DBG_DUMP

extern Func *CurrentFunc;

#endif

JsFunctionCodeGen *

NativeCodeGenerator::NewFunctionCodeGen(Js::FunctionBody *functionBody, Js::EntryPointInfo* info)

{

return HeapNewNoThrow(JsFunctionCodeGen, this, functionBody, info, this->IsInDebugMode());

}

JsLoopBodyCodeGen *

NativeCodeGenerator::NewLoopBodyCodeGen(Js::FunctionBody *functionBody, Js::EntryPointInfo* info)

{

return HeapNewNoThrow(JsLoopBodyCodeGen, this, functionBody, info, this->IsInDebugMode());

}

#ifdef ENABLE_PREJIT

bool

NativeCodeGenerator::DoBackEnd(Js::FunctionBody *fn)

{

if (PHASE_OFF(Js::BackEndPhase, fn))

{

return false;

}

if (fn->IsAsmJSModule() || fn->IsGeneratorAndJitIsDisabled())

{

return false;

}

return true;

}

void

NativeCodeGenerator::GenerateAllFunctions(Js::FunctionBody * fn)

{

Assert(IS_PREJIT_ON());

Assert(fn->GetDefaultFunctionEntryPointInfo()->entryPointIndex == 0);

// Make sure this isn't a deferred function

Assert(fn->GetFunctionBody() == fn);

Assert(!fn->IsDeferred());

if (DoBackEnd(fn))

{

if (fn->GetLoopCount() != 0 && fn->ForceJITLoopBody() && !IsInDebugMode())

{

// Only jit the loop body with /force:JITLoopBody

for (uint i = 0; i < fn->GetLoopCount(); i++)

{

Js::LoopHeader * loopHeader = fn->GetLoopHeader(i);

Js::EntryPointInfo * entryPointInfo = loopHeader->GetCurrentEntryPointInfo();

this->GenerateLoopBody(fn, loopHeader, entryPointInfo);

}

}

else

{

// A JIT attempt should have already been made through GenerateFunction

Assert(!fn->GetDefaultFunctionEntryPointInfo()->IsNotScheduled());

}

}

for (uint i = 0; i < fn->GetNestedCount(); i++)

{

Js::FunctionBody* functionToJIT = fn->GetNestedFunctionForExecution(i)->GetFunctionBody();

GenerateAllFunctions(functionToJIT);

}

}

#endif

#if _M_ARM

USHORT ArmExtractThumbImmediate16(PUSHORT address)

{

return ((address[0] << 12) & 0xf000) | // bits[15:12] in OP0[3:0]

((address[0] << 1) & 0x0800) | // bits[11] in OP0[10]

((address[1] >> 4) & 0x0700) | // bits[10:8] in OP1[14:12]

((address[1] >> 0) & 0x00ff); // bits[7:0] in OP1[7:0]

}

void ArmInsertThumbImmediate16(PUSHORT address, USHORT immediate)

{

USHORT opcode0;

USHORT opcode1;

opcode0 = address[0];

opcode1 = address[1];

opcode0 &= ~((0xf000 >> 12) | (0x0800 >> 1));

opcode1 &= ~((0x0700 << 4) | (0x00ff << 0));

opcode0 |= (immediate & 0xf000) >> 12; // bits[15:12] in OP0[3:0]

opcode0 |= (immediate & 0x0800) >> 1; // bits[11] in OP0[10]

opcode1 |= (immediate & 0x0700) << 4; // bits[10:8] in OP1[14:12]

opcode1 |= (immediate & 0x00ff) << 0; // bits[7:0] in OP1[7:0]

address[0] = opcode0;

address[1] = opcode1;

}

#endif

void DoFunctionRelocations(BYTE *function, DWORD functionOffset, DWORD functionSize, BYTE *module, size_t imageBase, IMAGE_SECTION_HEADER *textHeader, IMAGE_SECTION_HEADER *relocHeader)

{

PIMAGE_BASE_RELOCATION relocationBlock = (PIMAGE_BASE_RELOCATION)(module + relocHeader->PointerToRawData);

for (; relocationBlock->VirtualAddress > 0 && ((BYTE *)relocationBlock < (module + relocHeader->PointerToRawData + relocHeader->SizeOfRawData)); )

{

DWORD blockOffset = relocationBlock->VirtualAddress - textHeader->VirtualAddress;

// Skip relocation blocks that are before the function

if ((blockOffset + 0x1000) > functionOffset)

{

unsigned short *relocation = (unsigned short *)((unsigned char *)relocationBlock + sizeof(IMAGE_BASE_RELOCATION));

for (uint index = 0; index < ((relocationBlock->SizeOfBlock - sizeof(IMAGE_BASE_RELOCATION)) / 2); index++, relocation++)

{

int type = *relocation >> 12;

int offset = *relocation & 0xfff;

// If we are past the end of the function, we can stop.

if ((blockOffset + offset) >= (functionOffset + functionSize))

{

break;

}

if ((blockOffset + offset) < functionOffset)

{

continue;

}

switch (type)

{

case IMAGE_REL_BASED_ABSOLUTE:

break;

#if _M_IX86

case IMAGE_REL_BASED_HIGHLOW:

{

DWORD *patchAddrHL = (DWORD *) (function + blockOffset + offset - functionOffset);

DWORD patchAddrHLOffset = *patchAddrHL - imageBase - textHeader->VirtualAddress;

Assert((patchAddrHLOffset > functionOffset) && (patchAddrHLOffset < (functionOffset + functionSize)));

*patchAddrHL = patchAddrHLOffset - functionOffset + (DWORD)function;

}

break;

#elif defined(_M_X64_OR_ARM64)

case IMAGE_REL_BASED_DIR64:

{

ULONGLONG *patchAddr64 = (ULONGLONG *) (function + blockOffset + offset - functionOffset);

ULONGLONG patchAddr64Offset = *patchAddr64 - imageBase - textHeader->VirtualAddress;

Assert((patchAddr64Offset > functionOffset) && (patchAddr64Offset < (functionOffset + functionSize)));

*patchAddr64 = patchAddr64Offset - functionOffset + (ULONGLONG)function;

}

break;

#else

case IMAGE_REL_BASED_THUMB_MOV32:

{

USHORT *patchAddr = (USHORT *) (function + blockOffset + offset - functionOffset);

DWORD address = ArmExtractThumbImmediate16(patchAddr) | (ArmExtractThumbImmediate16(patchAddr + 2) << 16);

address = address - imageBase - textHeader->VirtualAddress - functionOffset + (DWORD)function;

ArmInsertThumbImmediate16(patchAddr, (USHORT)(address & 0xFFFF));

ArmInsertThumbImmediate16(patchAddr + 2, (USHORT)(address >> 16));

}

break;

#endif

default:

Assert(false);

break;

}

}

}

relocationBlock = (PIMAGE_BASE_RELOCATION) (((BYTE *) relocationBlock) + relocationBlock->SizeOfBlock);

}

}

class AutoRestoreDefaultEntryPoint

{

public:

AutoRestoreDefaultEntryPoint(Js::FunctionBody* functionBody):

functionBody(functionBody)

{

this->oldDefaultEntryPoint = functionBody->GetDefaultFunctionEntryPointInfo();

this->oldOriginalEntryPoint = functionBody->GetOriginalEntryPoint();

this->newEntryPoint = functionBody->CreateNewDefaultEntryPoint();

}

~AutoRestoreDefaultEntryPoint()

{

if (newEntryPoint && !newEntryPoint->IsCodeGenDone())

{

functionBody->RestoreOldDefaultEntryPoint(oldDefaultEntryPoint, oldOriginalEntryPoint, newEntryPoint);

}

}

private:

Js::FunctionBody* functionBody;

Js::FunctionEntryPointInfo* oldDefaultEntryPoint;

Js::JavascriptMethod oldOriginalEntryPoint;

Js::FunctionEntryPointInfo* newEntryPoint;

};

//static

void NativeCodeGenerator::Jit_TransitionFromSimpleJit(void *const framePointer)

{

TransitionFromSimpleJit(

Js::ScriptFunction::FromVar(Js::JavascriptCallStackLayout::FromFramePointer(framePointer)->functionObject));

}

//static

void NativeCodeGenerator::TransitionFromSimpleJit(Js::ScriptFunction *const function)

{

Assert(function);

Js::FunctionBody *const functionBody = function->GetFunctionBody();

Js::FunctionEntryPointInfo *const defaultEntryPointInfo = functionBody->GetDefaultFunctionEntryPointInfo();

if(defaultEntryPointInfo == functionBody->GetSimpleJitEntryPointInfo())

{

Assert(functionBody->GetExecutionMode() == ExecutionMode::SimpleJit);

Assert(function->GetFunctionEntryPointInfo() == defaultEntryPointInfo);

// The latest entry point is the simple JIT, transition to the next execution mode and schedule a full JIT

bool functionEntryPointUpdated = functionBody->GetScriptContext()->GetNativeCodeGenerator()->GenerateFunction(functionBody, function);

if (functionEntryPointUpdated)

{

// Transition to the next execution mode after scheduling a full JIT, in case of OOM before the entry point is changed

const bool transitioned = functionBody->TryTransitionToNextExecutionMode();

Assert(transitioned);

if (PHASE_TRACE(Js::SimpleJitPhase, functionBody))

{

wchar_t debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];

Output::Print(

L"SimpleJit (TransitionFromSimpleJit): function: %s (%s)",

functionBody->GetDisplayName(),

functionBody->GetDebugNumberSet(debugStringBuffer));

Output::Flush();

}

}

return;

}

if(function->GetFunctionEntryPointInfo() != defaultEntryPointInfo)

{

// A full JIT may have already been scheduled, or some entry point info got expired before the simple JIT entry point

// was ready. In any case, the function's entry point info is not the latest, so update it.

function->UpdateThunkEntryPoint(defaultEntryPointInfo, functionBody->GetDirectEntryPoint(defaultEntryPointInfo));

}

}

#ifdef IR_VIEWER

Js::Var

NativeCodeGenerator::RejitIRViewerFunction(Js::FunctionBody *fn, Js::ScriptContext *requestContext)

{

/* Note: adapted from NativeCodeGenerator::GenerateFunction (NativeCodeGenerator.cpp) */

Js::ScriptContext *scriptContext = fn->GetScriptContext();

PageAllocator *pageAllocator = scriptContext->GetThreadContext()->GetPageAllocator();

NativeCodeGenerator *nativeCodeGenerator = scriptContext->GetNativeCodeGenerator();

AutoRestoreDefaultEntryPoint autoRestore(fn);

Js::FunctionEntryPointInfo * entryPoint = fn->GetDefaultFunctionEntryPointInfo();

JsFunctionCodeGen workitem(this, fn, entryPoint, this->IsInDebugMode());

workitem.isRejitIRViewerFunction = true;

workitem.irViewerRequestContext = scriptContext;

workitem.SetJitMode(ExecutionMode::FullJit);

entryPoint->SetCodeGenPendingWithStackAllocatedWorkItem();

entryPoint->SetCodeGenQueued();

const auto recyclableData = GatherCodeGenData(fn, fn, entryPoint, &workitem);

workitem.SetRecyclableData(recyclableData);

nativeCodeGenerator->CodeGen(pageAllocator, &workitem, true);

return Js::CrossSite::MarshalVar(requestContext, workitem.GetIRViewerOutput(scriptContext));

}

#endif /* IR_VIEWER */

///----------------------------------------------------------------------------

///

/// NativeCodeGenerator::GenerateFunction

///

/// This is the main entry point for the runtime to call the native code

/// generator.

///

///----------------------------------------------------------------------------

bool

NativeCodeGenerator::GenerateFunction(Js::FunctionBody *fn, Js::ScriptFunction * function)

{

ASSERT_THREAD();

Assert(!fn->GetIsFromNativeCodeModule());

Assert(fn->GetScriptContext()->GetNativeCodeGenerator() == this);

Assert(fn->GetFunctionBody() == fn);

Assert(!fn->IsDeferred());

#if !defined(_M_ARM64)

if (fn->IsGeneratorAndJitIsDisabled())

{

// JITing generator functions is not complete nor stable yet so it is off by default.

// Also try/catch JIT support in generator functions is not a goal for threshold

// release so JITing generators containing try blocks is disabled for now.

return false;

}

if (IsInDebugMode() && fn->GetHasTry())

{

// Under debug mode disable JIT for functions that:

// - have try

return false;

}

#ifdef ENABLE_DEBUG_CONFIG_OPTIONS

if (Js::Configuration::Global.flags.Interpret &&

fn->GetDisplayName() &&

::wcsstr(Js::Configuration::Global.flags.Interpret, fn->GetDisplayName()))

{

return false;

}

#endif

if (fn->GetLoopCount() != 0 && fn->ForceJITLoopBody() && !IsInDebugMode())

{

// Don't code gen the function if the function has loop, ForceJITLoopBody is on,

// unless we are in debug mode in which case JIT loop body is disabled, even if it's forced.

return false;

}

// Create a work item with null entry point- we'll set it once its allocated

AutoPtr<JsFunctionCodeGen> workItemAutoPtr(this->NewFunctionCodeGen(fn, nullptr));

if ((JsFunctionCodeGen*) workItemAutoPtr == nullptr)

{

// OOM, just skip this work item and return.

return false;

}

Js::FunctionEntryPointInfo* entryPointInfo = nullptr;

if (function != nullptr)

{

entryPointInfo = fn->CreateNewDefaultEntryPoint();

}

else

{

entryPointInfo = fn->GetDefaultFunctionEntryPointInfo();

Assert(fn->IsInterpreterThunk() || fn->IsSimpleJitOriginalEntryPoint());

}

#ifdef ASMJS_PLAT

if (fn->GetIsAsmjsMode())

{

AnalysisAssert(function != nullptr);

Js::FunctionEntryPointInfo* oldFuncObjEntryPointInfo = (Js::FunctionEntryPointInfo*)function->GetEntryPointInfo();

Assert(oldFuncObjEntryPointInfo->GetIsAsmJSFunction()); // should be asmjs entrypoint info

// Set asmjs to be true in entrypoint

entryPointInfo->SetIsAsmJSFunction(true);

// Move the ModuleAddress from old Entrypoint to new entry point

entryPointInfo->SetModuleAddress(oldFuncObjEntryPointInfo->GetModuleAddress());

// Update the native address of the older entry point - this should be either the TJ entrypoint or the Interpreter Entry point

entryPointInfo->SetNativeAddress(oldFuncObjEntryPointInfo->address);

// have a reference to TJ entrypointInfo, this will be queued for collection in checkcodegen

entryPointInfo->SetOldFunctionEntryPointInfo(oldFuncObjEntryPointInfo);

Assert(PHASE_ON1(Js::AsmJsJITTemplatePhase) || (!oldFuncObjEntryPointInfo->GetIsTJMode() && !entryPointInfo->GetIsTJMode()));

// this changes the address in the entrypointinfo to be the AsmJsCodgenThunk

function->UpdateThunkEntryPoint(entryPointInfo, NativeCodeGenerator::CheckAsmJsCodeGenThunk);

if (PHASE_TRACE1(Js::AsmjsEntryPointInfoPhase))

Output::Print(L"New Entrypoint is CheckAsmJsCodeGenThunk for function: %s\n", fn->GetDisplayName());

}

else

#endif

{

fn->SetCheckCodeGenEntryPoint(entryPointInfo, NativeCodeGenerator::CheckCodeGenThunk);

if (function != nullptr)

{

function->UpdateThunkEntryPoint(entryPointInfo, NativeCodeGenerator::CheckCodeGenThunk);

}

}

JsFunctionCodeGen * workitem = workItemAutoPtr.Detach();

workitem->SetEntryPointInfo(entryPointInfo);

entryPointInfo->SetCodeGenPending(workitem);

InterlockedIncrement(&pendingCodeGenWorkItems);

if(!IS_PREJIT_ON())

{

workItems.LinkToEnd(workitem);

return true;

}

const ExecutionMode prejitJitMode = PrejitJitMode(fn);

workitem->SetJitMode(prejitJitMode);

try

{

AddToJitQueue(workitem, /*prioritize*/ true, /*lock*/ true, function);

}

catch (...)

{

// Add the item back to the list if AddToJitQueue throws. The position in the list is not important.

workitem->ResetJitMode();

workItems.LinkToEnd(workitem);

throw;

}

fn->TraceExecutionMode("Prejit (before)");

if(prejitJitMode == ExecutionMode::SimpleJit)

{

fn->TransitionToSimpleJitExecutionMode();

}

else

{

Assert(prejitJitMode == ExecutionMode::FullJit);

fn->TransitionToFullJitExecutionMode();

}

fn->TraceExecutionMode("Prejit");

Processor()->PrioritizeJobAndWait(this, entryPointInfo, function);

CheckCodeGenDone(fn, entryPointInfo, function);

return true;

#else

return false;

#endif

}

void NativeCodeGenerator::GenerateLoopBody(Js::FunctionBody * fn, Js::LoopHeader * loopHeader, Js::EntryPointInfo* entryPoint, uint localCount, Js::Var localSlots[])

{

ASSERT_THREAD();

Assert(fn->GetScriptContext()->GetNativeCodeGenerator() == this);

Assert(entryPoint->address == nullptr);

#if DBG_DUMP

if (PHASE_TRACE1(Js::JITLoopBodyPhase))

{

fn->DumpFunctionId(true);

Output::Print(L": %-20s LoopBody Start Loop: %2d ByteCode: %4d (%4d,%4d)\n", fn->GetDisplayName(), fn->GetLoopNumber(loopHeader),

loopHeader->endOffset - loopHeader->startOffset, loopHeader->startOffset, loopHeader->endOffset);

Output::Flush();

}

#endif

// If the parent function is JITted, no need to JIT this loop

// CanReleaseLoopHeaders is a quick and dirty way of checking if the

// function is currently being interpreted. If it is being interpreted,

// We'd still like to jit the loop body.

// We reset the interpretCount to 0 in case we switch back to the interpreter

if (fn->GetNativeEntryPointUsed() && fn->GetCanReleaseLoopHeaders() && (!fn->GetIsAsmJsFunction() || !(loopHeader->GetCurrentEntryPointInfo()->GetIsTJMode())))

{

loopHeader->ResetInterpreterCount();

return;

}

if (fn->GetIsAsmJsFunction())

{

Js::FunctionEntryPointInfo* functionEntryPointInfo = (Js::FunctionEntryPointInfo*) fn->GetDefaultEntryPointInfo();

Js::LoopEntryPointInfo* loopEntryPointInfo = (Js::LoopEntryPointInfo*)entryPoint;

loopEntryPointInfo->SetIsAsmJSFunction(true);

loopEntryPointInfo->SetModuleAddress(functionEntryPointInfo->GetModuleAddress());

}

JsLoopBodyCodeGen * workitem = this->NewLoopBodyCodeGen(fn, entryPoint);

if (!workitem)

{

// OOM, just skip this work item and return.

return;

}

entryPoint->SetCodeGenPending(workitem);

workitem->loopHeader = loopHeader;

try

{

if (!fn->GetIsAsmJsFunction()) // not needed for asmjs as we don't profile in asm mode

{

const uint profiledRegBegin = fn->GetConstantCount();

const uint profiledRegEnd = localCount;

if (profiledRegBegin < profiledRegEnd)

{

workitem->symIdToValueTypeMap =

HeapNew(JsLoopBodyCodeGen::SymIdToValueTypeMap, &HeapAllocator::Instance, profiledRegEnd - profiledRegBegin);

Recycler *recycler = fn->GetScriptContext()->GetRecycler();

for (uint i = profiledRegBegin; i < profiledRegEnd; i++)

{

if (localSlots[i] && IsValidVar(localSlots[i], recycler))

{

workitem->symIdToValueTypeMap->Add(i, ValueType::Uninitialized.Merge(localSlots[i]));

}

}

}

}

workitem->SetJitMode(ExecutionMode::FullJit);

AddToJitQueue(workitem, /*prioritize*/ true, /*lock*/ true);

}

catch (...)

{

// If adding to the JIT queue fails we need to revert the state of the entry point

// and delete the work item

entryPoint->RevertToNotScheduled();

workitem->Delete();

throw;

}

if (!Processor()->ProcessesInBackground() || fn->ForceJITLoopBody())

{

Processor()->PrioritizeJobAndWait(this, entryPoint);

}

}

bool

NativeCodeGenerator::IsValidVar(const Js::Var var, Recycler *const recycler)

{

using namespace Js;

Assert(var);

Assert(recycler);

// We may be handling uninitialized memory here, need to ensure that each recycler-allocated object is valid before it is

// read. Virtual functions shouldn't be called because the type ID may match by coincidence but the vtable can still be

// invalid, even if it is deemed to be a "valid" object, since that only validates that the memory is still owned by the

// recycler. This function validates the memory that ValueType::Merge(Var) reads.

if(TaggedInt::Is(var))

{

return true;

}

#if FLOATVAR

if(JavascriptNumber::Is_NoTaggedIntCheck(var))

{

return true;

}

#endif

RecyclableObject *const recyclableObject = RecyclableObject::FromVar(var);

if(!recycler->IsValidObject(recyclableObject, sizeof(*recyclableObject)))

{

return false;

}

INT_PTR vtable = VirtualTableInfoBase::GetVirtualTable(var);

if (vtable <= USHRT_MAX || (vtable & 1))

{

// Don't have a vtable, is it not a var, may be a frame display?

return false;

}

Type *const type = recyclableObject->GetType();

if(!recycler->IsValidObject(type, sizeof(*type)))

{

return false;

}

#if !FLOATVAR

if(JavascriptNumber::Is_NoTaggedIntCheck(var))

{

return true;

}

#endif

const TypeId typeId = type->GetTypeId();

if(typeId < static_cast<TypeId>(0))

{

return false;

}

if(!DynamicType::Is(typeId))

{

return true;

}

DynamicType *const dynamicType = static_cast<DynamicType *>(type);

if(!recycler->IsValidObject(dynamicType, sizeof(*dynamicType)))

{

return false;

}

DynamicTypeHandler *const typeHandler = dynamicType->GetTypeHandler();

if(!recycler->IsValidObject(typeHandler, sizeof(*typeHandler)))

{

return false;

}

// Not using DynamicObject::FromVar since there's a virtual call in there

DynamicObject *const object = static_cast<DynamicObject *>(recyclableObject);

if(!recycler->IsValidObject(object, sizeof(*object)))

{

return false;

}

if(typeId != TypeIds_Array)

{

ArrayObject* const objectArray = object->GetObjectArrayUnchecked();

return objectArray == nullptr || recycler->IsValidObject(objectArray, sizeof(*objectArray));

}

// Not using JavascriptArray::FromVar since there's a virtual call in there

JavascriptArray *const array = static_cast<JavascriptArray *>(object);

if(!recycler->IsValidObject(array, sizeof(*array)))

{

return false;

}

return true;

}

#if ENABLE_DEBUG_CONFIG_OPTIONS

volatile UINT_PTR NativeCodeGenerator::CodegenFailureSeed = 0;

#endif

void

NativeCodeGenerator::CodeGen(PageAllocator * pageAllocator, CodeGenWorkItem* workItem, const bool foreground)

{

if(foreground)

{

// Func::Codegen has a lot of things on the stack, so probe the stack here instead

PROBE_STACK(scriptContext, Js::Constants::MinStackJITCompile);

}

#if ENABLE_DEBUG_CONFIG_OPTIONS

if (!foreground && Js::Configuration::Global.flags.IsEnabled(Js::InduceCodeGenFailureFlag))

{

if (NativeCodeGenerator::CodegenFailureSeed == 0)

{

// Initialize the seed

NativeCodeGenerator::CodegenFailureSeed = Js::Configuration::Global.flags.InduceCodeGenFailureSeed;

if (NativeCodeGenerator::CodegenFailureSeed == 0)

{

LARGE_INTEGER ctr;

::QueryPerformanceCounter(&ctr);

NativeCodeGenerator::CodegenFailureSeed = ctr.HighPart ^ ctr.LowPart;

srand((uint)NativeCodeGenerator::CodegenFailureSeed);

}

}

int v = Math::Rand() % 100;

if (v < Js::Configuration::Global.flags.InduceCodeGenFailure)

{

switch (v % 3)

{

case 0: Js::Throw::OutOfMemory(); break;

case 1: throw Js::StackOverflowException(); break;

case 2: throw Js::OperationAbortedException(); break;

default:

Assert(false);

}

}

}

#endif

bool irviewerInstance = false;

#ifdef IR_VIEWER

irviewerInstance = true;

#endif

Assert(

workItem->Type() != JsFunctionType ||

irviewerInstance ||

IsThunk(workItem->GetFunctionBody()->GetDirectEntryPoint(workItem->GetEntryPoint())) ||

IsAsmJsCodeGenThunk(workItem->GetFunctionBody()->GetDirectEntryPoint(workItem->GetEntryPoint())));

InterlockedExchangeAdd(&this->byteCodeSizeGenerated, workItem->GetByteCodeCount()); // must be interlocked because this data may be modified in the foreground and background thread concurrently

Js::FunctionBody* body = workItem->GetFunctionBody();

int nRegs = body->GetLocalsCount();

AssertMsg((nRegs + 1) == (int)(SymID)(nRegs + 1), "SymID too small...");

CodeGenAllocators *const allocators =

foreground ? EnsureForegroundAllocators(pageAllocator) : GetBackgroundAllocator(pageAllocator); // okay to do outside lock since the respective function is called only from one thread

Js::ScriptContextProfiler *const codeGenProfiler =

#ifdef PROFILE_EXEC

foreground ? EnsureForegroundCodeGenProfiler() : GetBackgroundCodeGenProfiler(pageAllocator); // okay to do outside lock since the respective function is called only from one thread

#else

nullptr;

#endif

NoRecoverMemoryJitArenaAllocator funcAlloc(L"BE-FuncAlloc", pageAllocator, Js::Throw::OutOfMemory);

Js::ReadOnlyDynamicProfileInfo profileInfo(

body->HasDynamicProfileInfo() ? body->GetAnyDynamicProfileInfo() : nullptr,

foreground ? nullptr : &funcAlloc);

bool rejit;

ThreadContext *threadContext = scriptContext->GetThreadContext();

double startTime = threadContext->JITTelemetry.Now();

do

{

// the number allocator needs to be on the stack so that if we are doing foreground JIT

// the chunk allocated from the recycler will be stacked pinned

CodeGenNumberAllocator numberAllocator(

foreground? nullptr : scriptContext->GetThreadContext()->GetCodeGenNumberThreadAllocator(),

scriptContext->GetRecycler());

Func *func =

JitAnew(

(&funcAlloc),

Func,

(&funcAlloc),

workItem,

nullptr,

workItem->GetEntryPoint()->GetPolymorphicInlineCacheInfo()->GetSelfInfo(),

allocators,

&numberAllocator,

&profileInfo,

codeGenProfiler,

!foreground);

#if DBG_DUMP

CurrentFunc = func;

#endif

func->m_symTable->SetStartingID(static_cast<SymID>(nRegs + 1));

try

{

// Although we don't need to release the Arena memory, we need to invoke Func destructor.

// Use an auto object for it. Put it here to ensure "func" is cleared whenever we have an

// exception (RejitException or AbortException).

AutoAllocatorObjectPtr<Func, JitArenaAllocator> autoFunc(func, &funcAlloc);

func->Codegen();

rejit = false;

}

catch(Js::RejitException ex)

{

// The work item needs to be rejitted, likely due to some optimization that was too aggressive

if(ex.Reason() == RejitReason::AggressiveIntTypeSpecDisabled)

{

const bool isJitLoopBody = workItem->Type() == JsLoopBodyWorkItemType;

profileInfo.DisableAggressiveIntTypeSpec(isJitLoopBody);

if (body->HasDynamicProfileInfo())

{

body->GetAnyDynamicProfileInfo()->DisableAggressiveIntTypeSpec(isJitLoopBody);

}

}

else if(ex.Reason() == RejitReason::InlineApplyDisabled)

{

body->SetDisableInlineApply(true);

}

else if(ex.Reason() == RejitReason::InlineSpreadDisabled)

{

body->SetDisableInlineSpread(true);

}

else if(ex.Reason() == RejitReason::DisableSwitchOptExpectingInteger ||

ex.Reason() == RejitReason::DisableSwitchOptExpectingString)

{

profileInfo.DisableSwitchOpt();

if(body->HasDynamicProfileInfo())

{

body->GetAnyDynamicProfileInfo()->DisableSwitchOpt();

}

}

else

{

Assert(ex.Reason() == RejitReason::TrackIntOverflowDisabled);

profileInfo.DisableTrackCompoundedIntOverflow();

if(body->HasDynamicProfileInfo())

{

body->GetAnyDynamicProfileInfo()->DisableTrackCompoundedIntOverflow();

}

}

if(PHASE_TRACE(Js::ReJITPhase, body))

{

wchar_t debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];

Output::Print(

L"Rejit (compile-time): function: %s (%s) reason: %S\n",

body->GetDisplayName(),

body->GetDebugNumberSet(debugStringBuffer),

ex.ReasonName());

}

rejit = true;

funcAlloc.Reset();

if(!foreground)

{

profileInfo.OnBackgroundAllocatorReset();

}

}

// Either the entry point has a reference to the number now, or we failed to code gen and we

// don't need to numbers, we can flush the completed page now.

//

// If the number allocator is NULL then we are shutting down the thread context and so too the

// code generator. The number allocator must be freed before the recycler (and thus before the

// code generator) so we can't and don't need to flush it.

CodeGenNumberThreadAllocator * threadNumberAllocator = this->scriptContext->GetThreadContext()->GetCodeGenNumberThreadAllocator();

if (threadNumberAllocator != nullptr)

{

threadNumberAllocator->FlushAllocations();

}

} while(rejit);

threadContext->JITTelemetry.LogTime(threadContext->JITTelemetry.Now() - startTime);

#ifdef BGJIT_STATS

// Must be interlocked because the following data may be modified from the background and foreground threads concurrently

Js::ScriptContext *scriptContext = workItem->GetScriptContext();

if (workItem->Type() == JsFunctionType)

{

InterlockedExchangeAdd(&scriptContext->bytecodeJITCount, workItem->GetByteCodeCount());

InterlockedIncrement(&scriptContext->funcJITCount);

}

else if(workItem->Type() == JsLoopBodyWorkItemType)

{

InterlockedIncrement(&scriptContext->loopJITCount);

}

#endif

}

void NativeCodeGenerator::SetProfileMode(BOOL fSet)

{

this->SetNativeEntryPoint = fSet? Js::FunctionBody::ProfileSetNativeEntryPoint : Js::FunctionBody::DefaultSetNativeEntryPoint;

}

#if _M_IX86

__declspec(naked)

Js::Var

NativeCodeGenerator::CheckAsmJsCodeGenThunk(Js::RecyclableObject* function, Js::CallInfo callInfo, ...)

{

__asm

{