/*

* Copyright (C) 2003-2021 Apple Inc. All rights reserved.

* Copyright (C) 2007 Eric Seidel <eric@webkit.org>

*

* This library is free software; you can redistribute it and/or

* modify it under the terms of the GNU Lesser General Public

* License as published by the Free Software Foundation; either

* version 2 of the License, or (at your option) any later version.

*

* This library is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

* Lesser General Public License for more details.

*

* You should have received a copy of the GNU Lesser General Public

* License along with this library; if not, write to the Free Software

* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

*

*/

#include "config.h"

#include "Heap.h"

#include "BuiltinExecutables.h"

#include "CodeBlock.h"

#include "CodeBlockSetInlines.h"

#include "CollectingScope.h"

#include "ConservativeRoots.h"

#include "EdenGCActivityCallback.h"

#include "Exception.h"

#include "FullGCActivityCallback.h"

#include "FunctionExecutableInlines.h"

#include "GCActivityCallback.h"

#include "GCIncomingRefCountedInlines.h"

#include "GCIncomingRefCountedSetInlines.h"

#include "GCSegmentedArrayInlines.h"

#include "GCTypeMap.h"

#include "HasOwnPropertyCache.h"

#include "HeapHelperPool.h"

#include "HeapIterationScope.h"

#include "HeapProfiler.h"

#include "HeapSnapshot.h"

#include "HeapVerifier.h"

#include "IncrementalSweeper.h"

#include "Interpreter.h"

#include "IsoCellSetInlines.h"

#include "JITStubRoutineSet.h"

#include "JITWorklistInlines.h"

#include "JSFinalizationRegistry.h"

#include "JSVirtualMachineInternal.h"

#include "JSWeakMap.h"

#include "JSWeakObjectRef.h"

#include "JSWeakSet.h"

#include "JSWebAssemblyCodeBlock.h"

#include "MachineStackMarker.h"

#include "MarkStackMergingConstraint.h"

#include "MarkedJSValueRefArray.h"

#include "MarkedSpaceInlines.h"

#include "MarkingConstraintSet.h"

#include "PreventCollectionScope.h"

#include "SamplingProfiler.h"

#include "ShadowChicken.h"

#include "SpaceTimeMutatorScheduler.h"

#include "StochasticSpaceTimeMutatorScheduler.h"

#include "StopIfNecessaryTimer.h"

#include "SubspaceInlines.h"

#include "SuperSampler.h"

#include "SweepingScope.h"

#include "SymbolTableInlines.h"

#include "SynchronousStopTheWorldMutatorScheduler.h"

#include "TypeProfiler.h"

#include "TypeProfilerLog.h"

#include "VM.h"

#include "VerifierSlotVisitorInlines.h"

#include "WeakMapImplInlines.h"

#include "WeakSetInlines.h"

#include <algorithm>

#include <wtf/CryptographicallyRandomNumber.h>

#include <wtf/ListDump.h>

#include <wtf/RAMSize.h>

#include <wtf/Scope.h>

#include <wtf/SimpleStats.h>

#include <wtf/Threading.h>

#if USE(BMALLOC_MEMORY_FOOTPRINT_API)

#include <bmalloc/bmalloc.h>

#endif

#if USE(FOUNDATION)

#include <wtf/spi/cocoa/objcSPI.h>

#endif

#ifdef JSC_GLIB_API_ENABLED

#include "JSCGLibWrapperObject.h"

#endif

namespace JSC {

namespace {

static constexpr bool verboseStop = false;

double maxPauseMS(double thisPauseMS)

{

static double maxPauseMS;

maxPauseMS = std::max(thisPauseMS, maxPauseMS);

return maxPauseMS;

}

size_t minHeapSize(HeapType heapType, size_t ramSize)

{

if (heapType == LargeHeap) {

double result = std::min(

static_cast<double>(Options::largeHeapSize()),

ramSize * Options::smallHeapRAMFraction());

return static_cast<size_t>(result);

}

return Options::smallHeapSize();

}

size_t proportionalHeapSize(size_t heapSize, size_t ramSize)

{

if (VM::isInMiniMode())

return Options::miniVMHeapGrowthFactor() * heapSize;

#if USE(BMALLOC_MEMORY_FOOTPRINT_API)

size_t memoryFootprint = bmalloc::api::memoryFootprint();

if (memoryFootprint < ramSize * Options::smallHeapRAMFraction())

return Options::smallHeapGrowthFactor() * heapSize;

if (memoryFootprint < ramSize * Options::mediumHeapRAMFraction())

return Options::mediumHeapGrowthFactor() * heapSize;

#else

if (heapSize < ramSize * Options::smallHeapRAMFraction())

return Options::smallHeapGrowthFactor() * heapSize;

if (heapSize < ramSize * Options::mediumHeapRAMFraction())

return Options::mediumHeapGrowthFactor() * heapSize;

#endif

return Options::largeHeapGrowthFactor() * heapSize;

}

bool isValidSharedInstanceThreadState(VM& vm)

{

return vm.currentThreadIsHoldingAPILock();

}

bool isValidThreadState(VM& vm)

{

if (vm.atomStringTable() != Thread::current().atomStringTable())

return false;

if (vm.isSharedInstance() && !isValidSharedInstanceThreadState(vm))

return false;

return true;

}

void recordType(VM& vm, TypeCountSet& set, JSCell* cell)

{

const char* typeName = "[unknown]";

const ClassInfo* info = cell->classInfo(vm);

if (info && info->className)

typeName = info->className;

set.add(typeName);

}

bool measurePhaseTiming()

{

return false;

}

HashMap<const char*, GCTypeMap<SimpleStats>>& timingStats()

{

static HashMap<const char*, GCTypeMap<SimpleStats>>* result;

static std::once_flag once;

std::call_once(

once,

[] {

result = new HashMap<const char*, GCTypeMap<SimpleStats>>();

});

return *result;

}

SimpleStats& timingStats(const char* name, CollectionScope scope)

{

return timingStats().add(name, GCTypeMap<SimpleStats>()).iterator->value[scope];

}

class TimingScope {

public:

TimingScope(std::optional<CollectionScope> scope, const char* name)

: m_scope(scope)

, m_name(name)

{

if (measurePhaseTiming())

m_before = MonotonicTime::now();

}

TimingScope(Heap& heap, const char* name)

: TimingScope(heap.collectionScope(), name)

{

}

void setScope(std::optional<CollectionScope> scope)

{

m_scope = scope;

}

void setScope(Heap& heap)

{

setScope(heap.collectionScope());

}

~TimingScope()

{

if (measurePhaseTiming()) {

MonotonicTime after = MonotonicTime::now();

Seconds timing = after - m_before;

SimpleStats& stats = timingStats(m_name, *m_scope);

stats.add(timing.milliseconds());

dataLog("[GC:", *m_scope, "] ", m_name, " took: ", timing.milliseconds(), "ms (average ", stats.mean(), "ms).\n");

}

}

private:

std::optional<CollectionScope> m_scope;

MonotonicTime m_before;

const char* m_name;

};

} // anonymous namespace

class Heap::HeapThread final : public AutomaticThread {

public:

HeapThread(const AbstractLocker& locker, Heap& heap)

: AutomaticThread(locker, heap.m_threadLock, heap.m_threadCondition.copyRef())

, m_heap(heap)

{

}

const char* name() const final

{

return "JSC Heap Collector Thread";

}

private:

PollResult poll(const AbstractLocker& locker) final

{

if (m_heap.m_threadShouldStop) {

m_heap.notifyThreadStopping(locker);

return PollResult::Stop;

}

if (m_heap.shouldCollectInCollectorThread(locker)) {

m_heap.m_collectorThreadIsRunning = true;

return PollResult::Work;

}

m_heap.m_collectorThreadIsRunning = false;

return PollResult::Wait;

}

WorkResult work() final

{

m_heap.collectInCollectorThread();

return WorkResult::Continue;

}

void threadDidStart() final

{

Thread::registerGCThread(GCThreadType::Main);

}

void threadIsStopping(const AbstractLocker&) final

{

m_heap.m_collectorThreadIsRunning = false;

}

Heap& m_heap;

};

Heap::Heap(VM& vm, HeapType heapType)

: m_heapType(heapType)

, m_ramSize(Options::forceRAMSize() ? Options::forceRAMSize() : ramSize())

, m_minBytesPerCycle(minHeapSize(m_heapType, m_ramSize))

, m_maxEdenSize(m_minBytesPerCycle)

, m_maxHeapSize(m_minBytesPerCycle)

, m_objectSpace(this)

, m_machineThreads(makeUnique<MachineThreads>())

, m_collectorSlotVisitor(makeUnique<SlotVisitor>(*this, "C"))

, m_mutatorSlotVisitor(makeUnique<SlotVisitor>(*this, "M"))

, m_mutatorMarkStack(makeUnique<MarkStackArray>())

, m_raceMarkStack(makeUnique<MarkStackArray>())

, m_constraintSet(makeUnique<MarkingConstraintSet>(*this))

, m_handleSet(vm)

, m_codeBlocks(makeUnique<CodeBlockSet>())

, m_jitStubRoutines(makeUnique<JITStubRoutineSet>())

, m_vm(vm)

// We seed with 10ms so that GCActivityCallback::didAllocate doesn't continuously

// schedule the timer if we've never done a collection.

, m_fullActivityCallback(GCActivityCallback::tryCreateFullTimer(this))

, m_edenActivityCallback(GCActivityCallback::tryCreateEdenTimer(this))

, m_sweeper(adoptRef(*new IncrementalSweeper(this)))

, m_stopIfNecessaryTimer(adoptRef(*new StopIfNecessaryTimer(vm)))

, m_sharedCollectorMarkStack(makeUnique<MarkStackArray>())

, m_sharedMutatorMarkStack(makeUnique<MarkStackArray>())

, m_helperClient(&heapHelperPool())

, m_threadLock(Box<Lock>::create())

, m_threadCondition(AutomaticThreadCondition::create())

{

m_worldState.store(0);

for (unsigned i = 0, numberOfParallelThreads = heapHelperPool().numberOfThreads(); i < numberOfParallelThreads; ++i) {

std::unique_ptr<SlotVisitor> visitor = makeUnique<SlotVisitor>(*this, toCString("P", i + 1));

if (Options::optimizeParallelSlotVisitorsForStoppedMutator())

visitor->optimizeForStoppedMutator();

m_availableParallelSlotVisitors.append(visitor.get());

m_parallelSlotVisitors.append(WTFMove(visitor));

}

if (Options::useConcurrentGC()) {

if (Options::useStochasticMutatorScheduler())

m_scheduler = makeUnique<StochasticSpaceTimeMutatorScheduler>(*this);

else

m_scheduler = makeUnique<SpaceTimeMutatorScheduler>(*this);

} else {

// We simulate turning off concurrent GC by making the scheduler say that the world

// should always be stopped when the collector is running.

m_scheduler = makeUnique<SynchronousStopTheWorldMutatorScheduler>();

}

if (Options::verifyHeap())

m_verifier = makeUnique<HeapVerifier>(this, Options::numberOfGCCyclesToRecordForVerification());

m_collectorSlotVisitor->optimizeForStoppedMutator();

// When memory is critical, allow allocating 25% of the amount above the critical threshold before collecting.

size_t memoryAboveCriticalThreshold = static_cast<size_t>(static_cast<double>(m_ramSize) * (1.0 - Options::criticalGCMemoryThreshold()));

m_maxEdenSizeWhenCritical = memoryAboveCriticalThreshold / 4;

Locker locker { *m_threadLock };

m_thread = adoptRef(new HeapThread(locker, *this));

}

Heap::~Heap()

{

// Scribble m_worldState to make it clear that the heap has already been destroyed if we crash in checkConn

m_worldState.store(0xbadbeeffu);

forEachSlotVisitor(

[&] (SlotVisitor& visitor) {

visitor.clearMarkStacks();

});

m_mutatorMarkStack->clear();

m_raceMarkStack->clear();

for (WeakBlock* block : m_logicallyEmptyWeakBlocks)

WeakBlock::destroy(*this, block);

}

bool Heap::isPagedOut()

{

return m_objectSpace.isPagedOut();

}

void Heap::dumpHeapStatisticsAtVMDestruction()

{

unsigned counter = 0;

m_objectSpace.forEachBlock([&] (MarkedBlock::Handle* block) {

unsigned live = 0;

block->forEachCell([&] (size_t, HeapCell* cell, HeapCell::Kind) {

if (cell->isLive())

live++;

return IterationStatus::Continue;

});

dataLogLn("[", counter++, "] ", block->cellSize(), ", ", live, " / ", block->cellsPerBlock(), " ", static_cast<double>(live) / block->cellsPerBlock() * 100, "% ", block->attributes(), " ", block->subspace()->name());

block->forEachCell([&] (size_t, HeapCell* heapCell, HeapCell::Kind kind) {

if (heapCell->isLive() && kind == HeapCell::Kind::JSCell) {

auto* cell = static_cast<JSCell*>(heapCell);

if (cell->isObject())

dataLogLn(" ", JSValue((JSObject*)cell));

else

dataLogLn(" ", *cell);

}

return IterationStatus::Continue;

});

});

}

// The VM is being destroyed and the collector will never run again.

// Run all pending finalizers now because we won't get another chance.

void Heap::lastChanceToFinalize()

{

MonotonicTime before;

if (UNLIKELY(Options::logGC())) {

before = MonotonicTime::now();

dataLog("[GC<", RawPointer(this), ">: shutdown ");

}

m_isShuttingDown = true;

RELEASE_ASSERT(!m_vm.entryScope);

RELEASE_ASSERT(m_mutatorState == MutatorState::Running);

if (m_collectContinuouslyThread) {

{

Locker locker { m_collectContinuouslyLock };

m_shouldStopCollectingContinuously = true;

m_collectContinuouslyCondition.notifyOne();

}

m_collectContinuouslyThread->waitForCompletion();

}

dataLogIf(Options::logGC(), "1");

// Prevent new collections from being started. This is probably not even necessary, since we're not

// going to call into anything that starts collections. Still, this makes the algorithm more

// obviously sound.

m_isSafeToCollect = false;

dataLogIf(Options::logGC(), "2");

bool isCollecting;

{

Locker locker { *m_threadLock };

RELEASE_ASSERT(m_lastServedTicket <= m_lastGrantedTicket);

isCollecting = m_lastServedTicket < m_lastGrantedTicket;

}

if (isCollecting) {

dataLogIf(Options::logGC(), "...]\n");

// Wait for the current collection to finish.

waitForCollector(

[&] (const AbstractLocker&) -> bool {

RELEASE_ASSERT(m_lastServedTicket <= m_lastGrantedTicket);

return m_lastServedTicket == m_lastGrantedTicket;

});

dataLogIf(Options::logGC(), "[GC<", RawPointer(this), ">: shutdown ");

}

dataLogIf(Options::logGC(), "3");

RELEASE_ASSERT(m_requests.isEmpty());

RELEASE_ASSERT(m_lastServedTicket == m_lastGrantedTicket);

// Carefully bring the thread down.

bool stopped = false;

{

Locker locker { *m_threadLock };

stopped = m_thread->tryStop(locker);

m_threadShouldStop = true;

if (!stopped)

m_threadCondition->notifyOne(locker);

}

dataLogIf(Options::logGC(), "4");

if (!stopped)

m_thread->join();

dataLogIf(Options::logGC(), "5 ");

if (UNLIKELY(Options::dumpHeapStatisticsAtVMDestruction()))

dumpHeapStatisticsAtVMDestruction();

m_arrayBuffers.lastChanceToFinalize();

m_objectSpace.stopAllocatingForGood();

m_objectSpace.lastChanceToFinalize();

releaseDelayedReleasedObjects();

sweepAllLogicallyEmptyWeakBlocks();

m_objectSpace.freeMemory();

dataLogIf(Options::logGC(), (MonotonicTime::now() - before).milliseconds(), "ms]\n");

}

void Heap::releaseDelayedReleasedObjects()

{

#if USE(FOUNDATION) || defined(JSC_GLIB_API_ENABLED)

// We need to guard against the case that releasing an object can create more objects due to the

// release calling into JS. When those JS call(s) exit and all locks are being dropped we end up

// back here and could try to recursively release objects. We guard that with a recursive entry

// count. Only the initial call will release objects, recursive calls simple return and let the

// the initial call to the function take care of any objects created during release time.

// This also means that we need to loop until there are no objects in m_delayedReleaseObjects

// and use a temp Vector for the actual releasing.

if (!m_delayedReleaseRecursionCount++) {

while (!m_delayedReleaseObjects.isEmpty()) {

ASSERT(m_vm.currentThreadIsHoldingAPILock());

auto objectsToRelease = WTFMove(m_delayedReleaseObjects);

{

// We need to drop locks before calling out to arbitrary code.

JSLock::DropAllLocks dropAllLocks(m_vm);

#if USE(FOUNDATION)

void* context = objc_autoreleasePoolPush();

#endif

objectsToRelease.clear();

#if USE(FOUNDATION)

objc_autoreleasePoolPop(context);

#endif

}

}

}

m_delayedReleaseRecursionCount--;

#endif

}

void Heap::reportExtraMemoryAllocatedSlowCase(size_t size)

{

didAllocate(size);

collectIfNecessaryOrDefer();

}

void Heap::deprecatedReportExtraMemorySlowCase(size_t size)

{

// FIXME: Change this to use SaturatedArithmetic when available.

// https://bugs.webkit.org/show_bug.cgi?id=170411

CheckedSize checkedNewSize = m_deprecatedExtraMemorySize;

checkedNewSize += size;

m_deprecatedExtraMemorySize = UNLIKELY(checkedNewSize.hasOverflowed()) ? std::numeric_limits<size_t>::max() : checkedNewSize.value();

reportExtraMemoryAllocatedSlowCase(size);

}

bool Heap::overCriticalMemoryThreshold(MemoryThresholdCallType memoryThresholdCallType)

{

#if USE(BMALLOC_MEMORY_FOOTPRINT_API)

if (memoryThresholdCallType == MemoryThresholdCallType::Direct || ++m_percentAvailableMemoryCachedCallCount >= 100) {

m_overCriticalMemoryThreshold = bmalloc::api::percentAvailableMemoryInUse() > Options::criticalGCMemoryThreshold();

m_percentAvailableMemoryCachedCallCount = 0;

}

return m_overCriticalMemoryThreshold;

#else

UNUSED_PARAM(memoryThresholdCallType);

return false;

#endif

}

void Heap::reportAbandonedObjectGraph()

{

// Our clients don't know exactly how much memory they

// are abandoning so we just guess for them.

size_t abandonedBytes = static_cast<size_t>(0.1 * capacity());

// We want to accelerate the next collection. Because memory has just

// been abandoned, the next collection has the potential to

// be more profitable. Since allocation is the trigger for collection,

// we hasten the next collection by pretending that we've allocated more memory.

if (m_fullActivityCallback) {

m_fullActivityCallback->didAllocate(*this,

m_sizeAfterLastCollect - m_sizeAfterLastFullCollect + m_bytesAllocatedThisCycle + m_bytesAbandonedSinceLastFullCollect);

}

m_bytesAbandonedSinceLastFullCollect += abandonedBytes;

}

void Heap::protect(JSValue k)

{

ASSERT(k);

ASSERT(m_vm.currentThreadIsHoldingAPILock());

if (!k.isCell())

return;

m_protectedValues.add(k.asCell());

}

bool Heap::unprotect(JSValue k)

{

ASSERT(k);

ASSERT(m_vm.currentThreadIsHoldingAPILock());

if (!k.isCell())

return false;

return m_protectedValues.remove(k.asCell());

}

void Heap::addReference(JSCell* cell, ArrayBuffer* buffer)

{

if (m_arrayBuffers.addReference(cell, buffer)) {

collectIfNecessaryOrDefer();

didAllocate(buffer->gcSizeEstimateInBytes());

}

}

template<typename CellType, typename CellSet>

void Heap::finalizeMarkedUnconditionalFinalizers(CellSet& cellSet)

{

cellSet.forEachMarkedCell(

[&] (HeapCell* cell, HeapCell::Kind) {

static_cast<CellType*>(cell)->finalizeUnconditionally(vm());

});

}

void Heap::finalizeUnconditionalFinalizers()

{

vm().builtinExecutables()->finalizeUnconditionally();

finalizeMarkedUnconditionalFinalizers<FunctionExecutable>(vm().functionExecutableSpace.space);

finalizeMarkedUnconditionalFinalizers<SymbolTable>(vm().symbolTableSpace);

finalizeMarkedUnconditionalFinalizers<ExecutableToCodeBlockEdge>(vm().executableToCodeBlockEdgesWithFinalizers); // We run this before CodeBlock's unconditional finalizer since CodeBlock looks at the owner executable's installed CodeBlock in its finalizeUnconditionally.

vm().forEachCodeBlockSpace(

[&] (auto& space) {

this->finalizeMarkedUnconditionalFinalizers<CodeBlock>(space.set);

});

finalizeMarkedUnconditionalFinalizers<StructureRareData>(vm().structureRareDataSpace);

finalizeMarkedUnconditionalFinalizers<UnlinkedFunctionExecutable>(vm().unlinkedFunctionExecutableSpace.set);

if (vm().m_weakSetSpace)

finalizeMarkedUnconditionalFinalizers<JSWeakSet>(*vm().m_weakSetSpace);

if (vm().m_weakMapSpace)

finalizeMarkedUnconditionalFinalizers<JSWeakMap>(*vm().m_weakMapSpace);

if (vm().m_weakObjectRefSpace)

finalizeMarkedUnconditionalFinalizers<JSWeakObjectRef>(*vm().m_weakObjectRefSpace);

if (vm().m_errorInstanceSpace)

finalizeMarkedUnconditionalFinalizers<ErrorInstance>(*vm().m_errorInstanceSpace);

// FinalizationRegistries currently rely on serial finalization because they can post tasks to the deferredWorkTimer, which normally expects tasks to only be posted by the API lock holder.

if (vm().m_finalizationRegistrySpace)

finalizeMarkedUnconditionalFinalizers<JSFinalizationRegistry>(*vm().m_finalizationRegistrySpace);

#if ENABLE(WEBASSEMBLY)

if (vm().m_webAssemblyCodeBlockSpace)

finalizeMarkedUnconditionalFinalizers<JSWebAssemblyCodeBlock>(*vm().m_webAssemblyCodeBlockSpace);

#endif

}

void Heap::willStartIterating()

{

m_objectSpace.willStartIterating();

}

void Heap::didFinishIterating()

{

m_objectSpace.didFinishIterating();

}

void Heap::completeAllJITPlans()

{

if (!Options::useJIT())

return;

#if ENABLE(JIT)

JITWorklist::ensureGlobalWorklist().completeAllPlansForVM(m_vm);

#endif // ENABLE(JIT)

}

template<typename Visitor>

void Heap::iterateExecutingAndCompilingCodeBlocks(Visitor& visitor, const Function<void(CodeBlock*)>& func)

{

m_codeBlocks->iterateCurrentlyExecuting(func);

#if ENABLE(JIT)

if (Options::useJIT())

JITWorklist::ensureGlobalWorklist().iterateCodeBlocksForGC(visitor, m_vm, func);

#else

UNUSED_PARAM(visitor);

#endif // ENABLE(JIT)

}

template<typename Func, typename Visitor>

void Heap::iterateExecutingAndCompilingCodeBlocksWithoutHoldingLocks(Visitor& visitor, const Func& func)

{

Vector<CodeBlock*, 256> codeBlocks;

iterateExecutingAndCompilingCodeBlocks(visitor,

[&] (CodeBlock* codeBlock) {

codeBlocks.append(codeBlock);

});

for (CodeBlock* codeBlock : codeBlocks)

func(codeBlock);

}

void Heap::assertMarkStacksEmpty()

{

bool ok = true;

if (!m_sharedCollectorMarkStack->isEmpty()) {

dataLog("FATAL: Shared collector mark stack not empty! It has ", m_sharedCollectorMarkStack->size(), " elements.\n");

ok = false;

}

if (!m_sharedMutatorMarkStack->isEmpty()) {

dataLog("FATAL: Shared mutator mark stack not empty! It has ", m_sharedMutatorMarkStack->size(), " elements.\n");

ok = false;

}

forEachSlotVisitor(

[&] (SlotVisitor& visitor) {

if (visitor.isEmpty())

return;

dataLog("FATAL: Visitor ", RawPointer(&visitor), " is not empty!\n");

ok = false;

});

RELEASE_ASSERT(ok);

}

void Heap::gatherStackRoots(ConservativeRoots& roots)

{

m_machineThreads->gatherConservativeRoots(roots, *m_jitStubRoutines, *m_codeBlocks, m_currentThreadState, m_currentThread);

}

void Heap::gatherJSStackRoots(ConservativeRoots& roots)

{

#if ENABLE(C_LOOP)

m_vm.interpreter->cloopStack().gatherConservativeRoots(roots, *m_jitStubRoutines, *m_codeBlocks);

#else

UNUSED_PARAM(roots);

#endif

}

void Heap::gatherScratchBufferRoots(ConservativeRoots& roots)

{

#if ENABLE(DFG_JIT)

if (!Options::useJIT())

return;

m_vm.gatherScratchBufferRoots(roots);

m_vm.scanSideState(roots);

#else

UNUSED_PARAM(roots);

#endif

}

void Heap::beginMarking()

{

TimingScope timingScope(*this, "Heap::beginMarking");

m_jitStubRoutines->clearMarks();

m_objectSpace.beginMarking();

setMutatorShouldBeFenced(true);

}

void Heap::removeDeadCompilerWorklistEntries()

{

if (!Options::useJIT())

return;

#if ENABLE(JIT)

JITWorklist::ensureGlobalWorklist().removeDeadPlans(m_vm);

#endif // ENABLE(JIT)

}

struct GatherExtraHeapData : MarkedBlock::CountFunctor {

GatherExtraHeapData(VM& vm, HeapAnalyzer& analyzer)

: m_vm(vm)

, m_analyzer(analyzer)

{

}

IterationStatus operator()(HeapCell* heapCell, HeapCell::Kind kind) const

{

if (isJSCellKind(kind)) {

JSCell* cell = static_cast<JSCell*>(heapCell);

cell->methodTable(m_vm)->analyzeHeap(cell, m_analyzer);

}

return IterationStatus::Continue;

}

VM& m_vm;

HeapAnalyzer& m_analyzer;

};

void Heap::gatherExtraHeapData(HeapProfiler& heapProfiler)

{

if (auto* analyzer = heapProfiler.activeHeapAnalyzer()) {

HeapIterationScope heapIterationScope(*this);

GatherExtraHeapData functor(m_vm, *analyzer);

m_objectSpace.forEachLiveCell(heapIterationScope, functor);

}

}

struct RemoveDeadHeapSnapshotNodes : MarkedBlock::CountFunctor {

RemoveDeadHeapSnapshotNodes(HeapSnapshot& snapshot)

: m_snapshot(snapshot)

{

}

IterationStatus operator()(HeapCell* cell, HeapCell::Kind kind) const

{

if (isJSCellKind(kind))

m_snapshot.sweepCell(static_cast<JSCell*>(cell));

return IterationStatus::Continue;

}

HeapSnapshot& m_snapshot;

};

void Heap::removeDeadHeapSnapshotNodes(HeapProfiler& heapProfiler)

{

if (HeapSnapshot* snapshot = heapProfiler.mostRecentSnapshot()) {

HeapIterationScope heapIterationScope(*this);

RemoveDeadHeapSnapshotNodes functor(*snapshot);

m_objectSpace.forEachDeadCell(heapIterationScope, functor);

snapshot->shrinkToFit();

}

}

void Heap::updateObjectCounts()

{

if (m_collectionScope && m_collectionScope.value() == CollectionScope::Full)

m_totalBytesVisited = 0;

m_totalBytesVisitedThisCycle = bytesVisited();

m_totalBytesVisited += m_totalBytesVisitedThisCycle;

}

void Heap::endMarking()

{

forEachSlotVisitor(

[&] (SlotVisitor& visitor) {

visitor.reset();

});

assertMarkStacksEmpty();

RELEASE_ASSERT(m_raceMarkStack->isEmpty());

m_objectSpace.endMarking();

setMutatorShouldBeFenced(Options::forceFencedBarrier());

}

size_t Heap::objectCount()

{

return m_objectSpace.objectCount();

}

size_t Heap::extraMemorySize()

{

// FIXME: Change this to use SaturatedArithmetic when available.

// https://bugs.webkit.org/show_bug.cgi?id=170411

CheckedSize checkedTotal = m_extraMemorySize;

checkedTotal += m_deprecatedExtraMemorySize;

checkedTotal += m_arrayBuffers.size();

size_t total = UNLIKELY(checkedTotal.hasOverflowed()) ? std::numeric_limits<size_t>::max() : checkedTotal.value();

ASSERT(m_objectSpace.capacity() >= m_objectSpace.size());

return std::min(total, std::numeric_limits<size_t>::max() - m_objectSpace.capacity());

}

size_t Heap::size()

{

return m_objectSpace.size() + extraMemorySize();

}

size_t Heap::capacity()

{

return m_objectSpace.capacity() + extraMemorySize();

}

size_t Heap::protectedGlobalObjectCount()

{

size_t result = 0;

forEachProtectedCell(

[&] (JSCell* cell) {

if (cell->isObject() && asObject(cell)->isGlobalObject())

result++;

});

return result;

}

size_t Heap::globalObjectCount()

{

HeapIterationScope iterationScope(*this);

size_t result = 0;

m_objectSpace.forEachLiveCell(

iterationScope,

[&] (HeapCell* heapCell, HeapCell::Kind kind) -> IterationStatus {

if (!isJSCellKind(kind))

return IterationStatus::Continue;

JSCell* cell = static_cast<JSCell*>(heapCell);

if (cell->isObject() && asObject(cell)->isGlobalObject())

result++;

return IterationStatus::Continue;

});

return result;

}

size_t Heap::protectedObjectCount()

{

size_t result = 0;

forEachProtectedCell(

[&] (JSCell*) {

result++;

});

return result;

}

std::unique_ptr<TypeCountSet> Heap::protectedObjectTypeCounts()

{

std::unique_ptr<TypeCountSet> result = makeUnique<TypeCountSet>();

forEachProtectedCell(

[&] (JSCell* cell) {

recordType(vm(), *result, cell);

});

return result;

}

std::unique_ptr<TypeCountSet> Heap::objectTypeCounts()

{

std::unique_ptr<TypeCountSet> result = makeUnique<TypeCountSet>();

HeapIterationScope iterationScope(*this);

m_objectSpace.forEachLiveCell(

iterationScope,

[&] (HeapCell* cell, HeapCell::Kind kind) -> IterationStatus {

if (isJSCellKind(kind))

recordType(vm(), *result, static_cast<JSCell*>(cell));

return IterationStatus::Continue;

});

return result;

}

void Heap::deleteAllCodeBlocks(DeleteAllCodeEffort effort)

{

if (m_collectionScope && effort == DeleteAllCodeIfNotCollecting)

return;

VM& vm = m_vm;

PreventCollectionScope preventCollectionScope(*this);

// If JavaScript is running, it's not safe to delete all JavaScript code, since

// we'll end up returning to deleted code.

RELEASE_ASSERT(!vm.entryScope);

RELEASE_ASSERT(!m_collectionScope);

completeAllJITPlans();

vm.forEachScriptExecutableSpace(

[&] (auto& spaceAndSet) {

HeapIterationScope heapIterationScope(*this);

auto& set = spaceAndSet.set;

set.forEachLiveCell(

[&] (HeapCell* cell, HeapCell::Kind) {

ScriptExecutable* executable = static_cast<ScriptExecutable*>(cell);

executable->clearCode(set);

});

});

#if ENABLE(WEBASSEMBLY)

{

// We must ensure that we clear the JS call ICs from Wasm. Otherwise, Wasm will

// have no idea that we cleared the code from all of the Executables in the

// VM. This could leave Wasm in an inconsistent state where it has an IC that

// points into a CodeBlock that could be dead. The IC will still succeed because

// it uses a callee check, but then it will call into dead code.

HeapIterationScope heapIterationScope(*this);

if (vm.m_webAssemblyCodeBlockSpace) {

vm.m_webAssemblyCodeBlockSpace->forEachLiveCell([&] (HeapCell* cell, HeapCell::Kind kind) {

ASSERT_UNUSED(kind, kind == HeapCell::JSCell);

JSWebAssemblyCodeBlock* codeBlock = static_cast<JSWebAssemblyCodeBlock*>(cell);

codeBlock->clearJSCallICs(vm);

});

}

}

#endif

}

void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort)

{

if (m_collectionScope && effort == DeleteAllCodeIfNotCollecting)

return;

VM& vm = m_vm;

PreventCollectionScope preventCollectionScope(*this);

RELEASE_ASSERT(!m_collectionScope);

HeapIterationScope heapIterationScope(*this);

vm.unlinkedFunctionExecutableSpace.set.forEachLiveCell(

[&] (HeapCell* cell, HeapCell::Kind) {

UnlinkedFunctionExecutable* executable = static_cast<UnlinkedFunctionExecutable*>(cell);

executable->clearCode(vm);

});

}

void Heap::deleteUnmarkedCompiledCode()

{

vm().forEachScriptExecutableSpace([] (auto& space) { space.space.sweep(); });

// Sweeping must occur before deleting stubs, otherwise the stubs might still think they're alive as they get deleted.

// And CodeBlock destructor is assuming that CodeBlock gets destroyed before UnlinkedCodeBlock gets destroyed.

vm().forEachCodeBlockSpace([] (auto& space) { space.space.sweep(); });

m_jitStubRoutines->deleteUnmarkedJettisonedStubRoutines();

}

void Heap::addToRememberedSet(const JSCell* constCell)

{

JSCell* cell = const_cast<JSCell*>(constCell);

ASSERT(cell);

ASSERT(!Options::useConcurrentJIT() || !isCompilationThread());

m_barriersExecuted++;

if (m_mutatorShouldBeFenced) {

WTF::loadLoadFence();

if (!isMarked(cell)) {

// During a full collection a store into an unmarked object that had surivived past

// collections will manifest as a store to an unmarked PossiblyBlack object. If the

// object gets marked at some time after this then it will go down the normal marking

// path. So, we don't have to remember this object. We could return here. But we go

// further and attempt to re-white the object.

RELEASE_ASSERT(m_collectionScope && m_collectionScope.value() == CollectionScope::Full);

if (cell->atomicCompareExchangeCellStateStrong(CellState::PossiblyBlack, CellState::DefinitelyWhite) == CellState::PossiblyBlack) {

// Now we protect against this race:

//

// 1) Object starts out black + unmarked.