//-------------------------------------------------------------------------------------------------------

// Copyright (C) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.

//-------------------------------------------------------------------------------------------------------

#include "CommonMemoryPch.h"

template <typename TBlockAttributes>

SmallNormalHeapBlockT<TBlockAttributes> *

HeapBlock::AsNormalBlock()

{

Assert(IsAnyNormalBlock());

return static_cast<SmallNormalHeapBlockT<TBlockAttributes> *>(this);

}

template <typename TBlockAttributes>

SmallLeafHeapBlockT<TBlockAttributes> *

HeapBlock::AsLeafBlock()

{

Assert(IsLeafBlock());

return static_cast<SmallLeafHeapBlockT<TBlockAttributes> *>(this);

}

template <typename TBlockAttributes>

SmallFinalizableHeapBlockT<TBlockAttributes> *

HeapBlock::AsFinalizableBlock()

{

Assert(IsAnyFinalizableBlock());

return static_cast<SmallFinalizableHeapBlockT<TBlockAttributes> *>(this);

}

#ifdef RECYCLER_WRITE_BARRIER

template <typename TBlockAttributes>

SmallNormalWithBarrierHeapBlockT<TBlockAttributes> *

HeapBlock::AsNormalWriteBarrierBlock()

{

Assert(IsNormalWriteBarrierBlock());

return static_cast<SmallNormalWithBarrierHeapBlockT<TBlockAttributes> *>(this);

}

template <typename TBlockAttributes>

SmallFinalizableWithBarrierHeapBlockT<TBlockAttributes> *

HeapBlock::AsFinalizableWriteBarrierBlock()

{

Assert(IsFinalizableWriteBarrierBlock());

return static_cast<SmallFinalizableWithBarrierHeapBlockT<TBlockAttributes> *>(this);

}

#endif

void

HeapBlock::SetNeedOOMRescan(Recycler * recycler)

{

Assert(!this->IsLeafBlock());

this->needOOMRescan = true;

recycler->SetNeedOOMRescan();

}

//========================================================================================================

// SmallHeapBlock

//========================================================================================================

template <class TBlockAttributes>

size_t

SmallHeapBlockT<TBlockAttributes>::GetAllocPlusSize(uint objectCount)

{

// Small Heap Block Layout:

// TrackerData * [objectCount] (Optional)

// ObjectInfo [objectCount] (In reverse index order)

// <Small*HeapBlock>

size_t allocPlusSize = Math::Align<size_t>(sizeof(unsigned char) * objectCount, sizeof(size_t));

#ifdef PROFILE_RECYCLER_ALLOC

if (Recycler::DoProfileAllocTracker())

{

allocPlusSize += objectCount * sizeof(void *);

}

#endif

return allocPlusSize;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ConstructorCommon(HeapBucket * bucket, ushort objectSize, ushort objectCount, HeapBlockType heapBlockType)

{

this->heapBucket = bucket;

this->Init(objectSize, objectCount);

Assert(heapBlockType < HeapBlock::HeapBlockType::SmallAllocBlockTypeCount + HeapBlock::HeapBlockType::MediumAllocBlockTypeCount);

Assert(objectCount > 1 && objectCount == (this->GetPageCount() * AutoSystemInfo::PageSize) / objectSize);

#ifdef RECYCLER_SLOW_CHECK_ENABLED

heapBucket->heapInfo->heapBlockCount[heapBlockType]++;

#endif

if (TBlockAttributes::IsSmallBlock)

{

Assert(heapBlockType < HeapBlockType::SmallAllocBlockTypeCount);

}

else

{

Assert(heapBlockType >= HeapBlockType::SmallAllocBlockTypeCount && heapBlockType < HeapBlockType::SmallBlockTypeCount);

}

DebugOnly(lastUncollectedAllocBytes = 0);

}

template <class TBlockAttributes>

SmallHeapBlockT<TBlockAttributes>::SmallHeapBlockT(HeapBucket * bucket, ushort objectSize, ushort objectCount, HeapBlockType heapBlockType)

: HeapBlock(heapBlockType),

bucketIndex(HeapInfo::GetBucketIndex(objectSize)),

validPointers(HeapInfo::smallAllocValidPointersMap.GetValidPointersForIndex(HeapInfo::GetBucketIndex(objectSize))),

objectSize(objectSize), objectCount(objectCount)

{

ConstructorCommon(bucket, objectSize, objectCount, heapBlockType);

}

template <>

SmallHeapBlockT<MediumAllocationBlockAttributes>::SmallHeapBlockT(HeapBucket * bucket, ushort objectSize, ushort objectCount, HeapBlockType heapBlockType)

: HeapBlock((HeapBlockType)(heapBlockType)),

bucketIndex(HeapInfo::GetMediumBucketIndex(objectSize)),

validPointers(HeapInfo::mediumAllocValidPointersMap.GetValidPointersForIndex(HeapInfo::GetMediumBucketIndex(objectSize))),

objectSize(objectSize), objectCount(objectCount)

{

ConstructorCommon(bucket, objectSize, objectCount, heapBlockType);

}

template <class TBlockAttributes>

SmallHeapBlockT<TBlockAttributes>::~SmallHeapBlockT()

{

Assert((this->segment == nullptr && this->address == nullptr) ||

(this->IsLeafBlock()) ||

this->GetPageAllocator(heapBucket->heapInfo->recycler)->IsClosed());

#ifdef RECYCLER_SLOW_CHECK_ENABLED

heapBucket->heapInfo->heapBlockCount[this->GetHeapBlockType()]--;

heapBucket->heapBlockCount--;

#endif

}

template <class TBlockAttributes>

uint

SmallHeapBlockT<TBlockAttributes>::GetObjectBitDeltaForBucketIndex(uint bucketIndex)

{

return bucketIndex + 1;

}

template <>

uint

SmallHeapBlockT<MediumAllocationBlockAttributes>::GetObjectBitDeltaForBucketIndex(uint bucketIndex)

{

return HeapInfo::GetObjectSizeForBucketIndex<MediumAllocationBlockAttributes>(bucketIndex) / HeapConstants::ObjectGranularity;

}

template <class TBlockAttributes>

uint

SmallHeapBlockT<TBlockAttributes>::GetPageCount() const

{

return TBlockAttributes::PageCount;

}

template <>

uint

SmallHeapBlockT<MediumAllocationBlockAttributes>::GetUnusablePageCount()

{

return ((MediumAllocationBlockAttributes::PageCount * AutoSystemInfo::PageSize) % this->objectSize) / AutoSystemInfo::PageSize;

}

template <>

void

SmallHeapBlockT<MediumAllocationBlockAttributes>::ProtectUnusablePages()

{

size_t count = this->GetUnusablePageCount();

if (count > 0)

{

char* startPage = this->address + (MediumAllocationBlockAttributes::PageCount - count) * AutoSystemInfo::PageSize;

DWORD oldProtect;

BOOL ret = ::VirtualProtect(startPage, count * AutoSystemInfo::PageSize, PAGE_READONLY, &oldProtect);

Assert(ret && oldProtect == PAGE_READWRITE);

::ResetWriteWatch(startPage, count*AutoSystemInfo::PageSize);

}

}

template <>

void

SmallHeapBlockT<MediumAllocationBlockAttributes>::RestoreUnusablePages()

{

size_t count = this->GetUnusablePageCount();

if (count > 0)

{

char* startPage = (char*)this->address + (MediumAllocationBlockAttributes::PageCount - count) * AutoSystemInfo::PageSize;

DWORD oldProtect;

BOOL ret = ::VirtualProtect(startPage, count * AutoSystemInfo::PageSize, PAGE_READWRITE, &oldProtect);

#if DBG

HeapBlock* block = this->heapBucket->heapInfo->recycler->heapBlockMap.GetHeapBlock(this->address);

// only need to do this after the unusable page is already successfully protected

// currently we don't have a flag to save that, but it should not fail after it successfully added to blockmap (see SetPage() implementation)

if (block)

{

Assert(block == this);

Assert(ret && oldProtect == PAGE_READONLY);

}

#endif

}

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ClearObjectInfoList()

{

ushort count = this->objectCount;

// the object info list is prefix to the object

memset(((byte *)this) - count, 0, count);

}

template <class TBlockAttributes>

byte&

SmallHeapBlockT<TBlockAttributes>::ObjectInfo(uint index)

{

// See SmallHeapBlockT<TBlockAttributes>::GetAllocPlusSize for layout description

// the object info list is prefix to the object and in reverse index order

Assert(index < this->objectCount);

return *(((byte *)this) - index - 1);

}

template <class TBlockAttributes>

ushort

SmallHeapBlockT<TBlockAttributes>::GetExpectedFreeObjectCount() const

{

Assert(this->GetRecycler()->IsSweeping());

return objectCount - markCount;

}

template <class TBlockAttributes>

uint

SmallHeapBlockT<TBlockAttributes>::GetExpectedFreeBytes() const

{

return GetExpectedFreeObjectCount() * objectSize;

}

template <class TBlockAttributes>

ushort

SmallHeapBlockT<TBlockAttributes>::GetExpectedSweepObjectCount() const

{

return GetExpectedFreeObjectCount() - freeCount;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::Init(ushort objectSize, ushort objectCount)

{

Assert(objectCount != 0);

Assert(TBlockAttributes::IsAlignedObjectSize(objectSize));

Assert(this->next == nullptr);

Assert(this->freeObjectList == nullptr);

Assert(this->freeCount == 0);

#if ENABLE_PARTIAL_GC

this->oldFreeCount = this->lastFreeCount = this->objectCount;

#else

this->lastFreeCount = this->objectCount;

#endif

#if ENABLE_CONCURRENT_GC

this->isPendingConcurrentSweep = false;

#endif

Assert(!this->isInAllocator);

Assert(!this->isClearedFromAllocator);

Assert(!this->isIntegratedBlock);

}

template <class TBlockAttributes>

BOOL

SmallHeapBlockT<TBlockAttributes>::ReassignPages(Recycler * recycler)

{

Assert(this->address == nullptr);

Assert(this->segment == nullptr);

PageSegment * segment;

auto pageAllocator = this->GetPageAllocator(recycler);

uint pagecount = this->GetPageCount();

char * address = pageAllocator->AllocPagesPageAligned(pagecount, &segment);

if (address == NULL)

{

return FALSE;

}

#if ENABLE_PARTIAL_GC

recycler->autoHeap.uncollectedNewPageCount += this->GetPageCount();

#endif

#ifdef RECYCLER_ZERO_MEM_CHECK

if (!this->IsLeafBlock()

#ifdef RECYCLER_WRITE_BARRIER_ALLOC_THREAD_PAGE

&& !this->IsWithBarrier()

#endif

)

{

recycler->VerifyZeroFill(address, AutoSystemInfo::PageSize * this->GetPageCount());

}

#endif

if (!this->SetPage(address, segment, recycler))

{

this->GetPageAllocator(recycler)->SuspendIdleDecommit();

this->ReleasePages(recycler);

this->GetPageAllocator(recycler)->ResumeIdleDecommit();

return FALSE;

}

RECYCLER_PERF_COUNTER_ADD(FreeObjectSize, this->GetPageCount() * AutoSystemInfo::PageSize);

RECYCLER_PERF_COUNTER_ADD(SmallHeapBlockFreeObjectSize, this->GetPageCount() * AutoSystemInfo::PageSize);

return TRUE;

}

template <class TBlockAttributes>

BOOL

SmallHeapBlockT<TBlockAttributes>::SetPage(__in_ecount_pagesize char * baseAddress, PageSegment * pageSegment, Recycler * recycler)

{

char* address = baseAddress;

Assert(HeapBlockMap32::GetLevel2Id(address) + (TBlockAttributes::PageCount - 1) < 256);

this->segment = pageSegment;

this->address = address;

// Set up the page to have nothing is free

Assert(this->freeObjectList == nullptr);

Assert(this->IsFreeBitsValid());

Assert(this->freeCount == 0);

Assert(this->freeCount == this->GetFreeBitVector()->Count());

Assert(this->objectCount == this->lastFreeCount);

Assert(this->explicitFreeBits.Count() == 0);

#if ENABLE_CONCURRENT_GC

Assert(recycler->IsConcurrentMarkState() || !recycler->IsMarkState() || recycler->IsCollectionDisabled());

#else

Assert(!recycler->IsMarkState() || recycler->IsCollectionDisabled());

#endif

Assert(this->bucketIndex <= 0xFF);

// We use the block type directly here, without the getter so that we can tell on the heap block map,

// whether the block is a medium block or not

if (!recycler->heapBlockMap.SetHeapBlock(this->address, this->GetPageCount() - this->GetUnusablePageCount(), this, this->heapBlockType, (byte)this->bucketIndex))

{

return FALSE;

}

// Retrieve pointer to mark bits for this block and store it locally.

// Note, mark bits aren't guaranteed to exist until after we register with HBM.

this->markBits = recycler->heapBlockMap.GetMarkBitVectorForPages<TBlockAttributes::BitVectorCount>(this->address);

Assert(this->markBits);

#if defined(_M_ARM32_OR_ARM64)

// We need to ensure that the above writes to the SmallHeapBlock are visible to the background GC thread.

// In particular, see Threshold 331596 -- we were seeing an old value for SmallHeapBlockT<TBlockAttributes>::markBits in ResetMarks.

// which caused the bit vector Copy operation there to AV.

// See also SmallHeapBlockT<TBlockAttributes>::ResetMarks.

MemoryBarrier();

#endif

this->ProtectUnusablePages();

return TRUE;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ReleasePages(Recycler * recycler)

{

Assert(recycler->collectionState != CollectionStateMark);

Assert(segment != nullptr);

Assert(address != nullptr);

#if DBG

if (this->IsLeafBlock())

{

RecyclerVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("Releasing leaf block pages at address 0x%p\n"), address);

}

#endif

char* address = this->address;

#ifdef RECYCLER_FREE_MEM_FILL

memset(address, DbgMemFill, AutoSystemInfo::PageSize * (this->GetPageCount()-this->GetUnusablePageCount()));

#endif

if (this->GetUnusablePageCount() > 0)

{

this->RestoreUnusablePages();

}

this->GetPageAllocator(recycler)->ReleasePages(address, this->GetPageCount(), this->GetPageSegment());

this->segment = nullptr;

this->address = nullptr;

}

#if ENABLE_BACKGROUND_PAGE_FREEING

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::BackgroundReleasePagesSweep(Recycler* recycler)

{

recycler->heapBlockMap.ClearHeapBlock(address, this->GetPageCount() - this->GetUnusablePageCount());

char* address = this->address;

if (this->GetUnusablePageCount() > 0)

{

this->RestoreUnusablePages();

}

this->GetPageAllocator(recycler)->BackgroundReleasePages(address, this->GetPageCount(), this->GetPageSegment());

this->address = nullptr;

this->segment = nullptr;

this->Reset();

}

#endif

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ReleasePagesShutdown(Recycler * recycler)

{

#if DBG

if (this->IsLeafBlock())

{

RecyclerVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("Releasing leaf block pages at address 0x%p\n"), address);

}

RemoveFromHeapBlockMap(recycler);

// Don't release the page in shut down, the page allocator will release them faster

// Leaf block's allocator need not be closed

Assert(this->IsLeafBlock() || this->GetPageAllocator(recycler)->IsClosed());

#endif

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::RemoveFromHeapBlockMap(Recycler* recycler)

{

recycler->heapBlockMap.ClearHeapBlock(address, this->GetPageCount() - this->GetUnusablePageCount());

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ReleasePagesSweep(Recycler * recycler)

{

RemoveFromHeapBlockMap(recycler);

ReleasePages(recycler);

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::Reset()

{

this->GetFreeBitVector()->ClearAll();

this->freeCount = 0;

this->markCount = 0;

#if ENABLE_PARTIAL_GC

this->oldFreeCount = this->lastFreeCount = this->objectCount;

#else

this->lastFreeCount = this->objectCount;

#endif

this->freeObjectList = nullptr;

this->lastFreeObjectHead = nullptr;

this->ClearObjectInfoList();

this->isInAllocator = false;

#if DBG || defined(RECYCLER_STATS)

this->GetDebugFreeBitVector()->ClearAll();

#endif

#if DBG

this->isClearedFromAllocator = false;

this->isIntegratedBlock = false;

#endif

// There is no page associated with this heap block,

// and therefore we should have no mark bits either

this->markBits = nullptr;

Assert(this->explicitFreeBits.Count() == 0);

}

// Map any object address to it's object index within the heap block

template <class TBlockAttributes>

ushort

SmallHeapBlockT<TBlockAttributes>::GetAddressIndex(void * objectAddress)

{

Assert(objectAddress >= address && objectAddress < this->GetEndAddress());

Assert(HeapInfo::IsAlignedAddress(objectAddress));

Assert(HeapInfo::IsAlignedAddress(address));

unsigned int offset = (unsigned int)((char*)objectAddress - address);

offset = offset >> HeapConstants::ObjectAllocationShift;

ushort index = validPointers.GetAddressIndex(offset);

Assert(index == SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit ||

index <= TBlockAttributes::MaxAddressBit);

return index;

}

template <class TBlockAttributes>

typename SmallHeapBlockT<TBlockAttributes>::SmallHeapBlockBitVector const*

SmallHeapBlockT<TBlockAttributes>::GetInvalidBitVector()

{

return HeapInfo::GetInvalidBitVector<TBlockAttributes>(objectSize);

}

template <class TBlockAttributes>

typename SmallHeapBlockT<TBlockAttributes>::BlockInfo const*

SmallHeapBlockT<TBlockAttributes>::GetBlockInfo()

{

return HeapInfo::GetBlockInfo<TBlockAttributes>(objectSize);

}

template <class TBlockAttributes>

ushort

SmallHeapBlockT<TBlockAttributes>::GetInteriorAddressIndex(void * interiorAddress)

{

Assert(interiorAddress >= address && interiorAddress < this->GetEndAddress());

Assert(HeapInfo::IsAlignedAddress(address));

unsigned int offset = (unsigned int)((char*)interiorAddress - address);

offset = offset >> HeapConstants::ObjectAllocationShift;

ushort index = validPointers.GetInteriorAddressIndex(offset);

Assert(index == SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit ||

index <= TBlockAttributes::MaxAddressBit);

return index;

}

template <class TBlockAttributes>

BOOL

SmallHeapBlockT<TBlockAttributes>::IsInFreeObjectList(void * objectAddress)

{

FreeObject * freeObject = this->freeObjectList;

while (freeObject != nullptr)

{

if (freeObject == objectAddress)

{

return true;

}

freeObject = freeObject->GetNext();

}

return false;

}

template <class TBlockAttributes>

template <typename TBlockType>

bool

SmallHeapBlockT<TBlockAttributes>::FindHeapObjectImpl(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject)

{

if (flags & FindHeapObjectFlags_AllowInterior)

{

objectAddress = (void*) this->GetRealAddressFromInterior(objectAddress);

if (objectAddress == nullptr)

{

return false;

}

}

ushort index = GetAddressIndex(objectAddress);

Assert(index != SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit);

if (index == SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit)

{

return false;

}

// If we have pending object, we still need to check the free bit if the caller requested the attribute to be correct

bool const disableCheck = ((flags & FindHeapObjectFlags_NoFreeBitVerify) != 0) ||

((flags & FindHeapObjectFlags_VerifyFreeBitForAttribute) != 0 && !this->HasPendingDisposeObjects());

if (!disableCheck)

{

// REVIEW: Checking if an object if free is strictly not necessary

// In all case, we should have a valid object, For memory protect heap, this is just to make sure we don't

// free pointers that are invalid.

#if ENABLE_CONCURRENT_GC

if (recycler->IsConcurrentSweepExecutingState())

{

// TODO: unless we know the state of the heap block, we don't know.

// skip the check for now.

}

else

#endif

{

if (flags & FindHeapObjectFlags_ClearedAllocators)

{

// Heap enum has some case where it allocates, so we can't assert

Assert(((HeapBucketT<TBlockType> *)this->heapBucket)->AllocatorsAreEmpty() || recycler->isHeapEnumInProgress);

}

else if (this->IsInAllocator())

{

((HeapBucketT<TBlockType> *)this->heapBucket)->UpdateAllocators();

}

// REVIEW allocation heuristics

if (this->EnsureFreeBitVector()->Test(this->GetObjectBitDelta() * index))

{

return false;

}

}

}

byte& attributes = ObjectInfo(index);

heapObject = RecyclerHeapObjectInfo(objectAddress, recycler, this, &attributes);

return true;

}

template <class TBlockAttributes>

BOOL

SmallHeapBlockT<TBlockAttributes>::IsValidObject(void* objectAddress)

{

if (objectAddress < this->GetAddress() || objectAddress >= this->GetEndAddress())

{

return false;

}

ushort index = GetAddressIndex(objectAddress);

if (index == SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit)

{

return false;

}

#if DBG

return !this->GetDebugFreeBitVector()->Test(GetAddressBitIndex(objectAddress));

#else

return true;

#endif

}

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::IsInAllocator() const

{

return isInAllocator;

}

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::HasPendingDisposeObjects()

{

return this->IsAnyFinalizableBlock() && this->AsFinalizableBlock<TBlockAttributes>()->HasPendingDisposeObjects();

}

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::HasAnyDisposeObjects()

{

return this->IsAnyFinalizableBlock() && this->AsFinalizableBlock<TBlockAttributes>()->HasAnyDisposeObjects();

}

template <class TBlockAttributes>

Recycler *

SmallHeapBlockT<TBlockAttributes>::GetRecycler() const

{

#if DBG

return this->heapBucket->heapInfo->recycler;

#else

return nullptr;

#endif

}

#if DBG

template <class TBlockAttributes>

BOOL

SmallHeapBlockT<TBlockAttributes>::IsFreeObject(void * objectAddress)

{

if (objectAddress < this->GetAddress() || objectAddress >= this->GetEndAddress())

{

return false;

}

ushort index = GetAddressIndex(objectAddress);

if (index == SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit)

{

return false;

}

return this->GetDebugFreeBitVector()->Test(GetAddressBitIndex(objectAddress));

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::VerifyMarkBitVector()

{

this->GetRecycler()->heapBlockMap.template VerifyMarkCountForPages<TBlockAttributes::BitVectorCount>(this->address, TBlockAttributes::PageCount);

}

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::IsClearedFromAllocator() const

{

return isClearedFromAllocator;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::SetIsClearedFromAllocator(bool value)

{

isClearedFromAllocator = value;

}

#endif

template <class TBlockAttributes>

byte *

SmallHeapBlockT<TBlockAttributes>::GetRealAddressFromInterior(void * interiorAddress)

{

Assert(interiorAddress >= this->address && interiorAddress < this->address + AutoSystemInfo::PageSize * this->GetPageCount());

ushort index = GetInteriorAddressIndex(interiorAddress);

if (index != SmallHeapBlockT<TBlockAttributes>::InvalidAddressBit)

{

return (byte *)this->address + index * this->GetObjectSize();

}

return nullptr;

}

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::TestObjectMarkedBit(void* objectAddress)

{

Assert(this->address != nullptr);

Assert(this->segment != nullptr);

uint bitIndex = GetAddressBitIndex(objectAddress);

Assert(IsValidBitIndex(bitIndex));

return this->GetMarkedBitVector()->Test(bitIndex) != 0;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::SetObjectMarkedBit(void* objectAddress)

{

Assert(this->address != nullptr);

Assert(this->segment != nullptr);

uint bitIndex = GetAddressBitIndex(objectAddress);

Assert(IsValidBitIndex(bitIndex));

this->GetMarkedBitVector()->Set(bitIndex);

}

#ifdef RECYCLER_MEMORY_VERIFY

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::SetExplicitFreeBitForObject(void* objectAddress)

{

Assert(this->address != nullptr);

Assert(this->segment != nullptr);

uint bitIndex = GetAddressBitIndex(objectAddress);

Assert(IsValidBitIndex(bitIndex));

BOOLEAN wasSet = this->explicitFreeBits.TestAndSet(bitIndex);

Assert(!wasSet);

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ClearExplicitFreeBitForObject(void* objectAddress)

{

Assert(this->address != nullptr);

Assert(this->segment != nullptr);

uint bitIndex = GetAddressBitIndex(objectAddress);

Assert(IsValidBitIndex(bitIndex));

BOOLEAN wasSet = this->explicitFreeBits.TestAndClear(bitIndex);

Assert(wasSet);

}

#endif

#ifdef RECYCLER_VERIFY_MARK

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::VerifyMark()

{

Assert(!this->needOOMRescan);

SmallHeapBlockBitVector * marked = this->GetMarkedBitVector();

SmallHeapBlockBitVector tempFreeBits;

this->BuildFreeBitVector(&tempFreeBits);

SmallHeapBlockBitVector * free = &tempFreeBits;

SmallHeapBlockBitVector const * invalid = this->GetInvalidBitVector();

uint objectWordCount = this->GetObjectWordCount();

Recycler * recycler = this->heapBucket->heapInfo->recycler;

FOREACH_BITSET_IN_FIXEDBV(bitIndex, marked)

{

if (!free->Test(bitIndex) && !invalid->Test(bitIndex))

{

Assert(IsValidBitIndex(bitIndex));

uint objectIndex = GetObjectIndexFromBitIndex((ushort)bitIndex);

Assert((this->ObjectInfo(objectIndex) & NewTrackBit) == 0);

// NOTE: We can't verify mark for software write barrier blocks, because they may have

// non-pointer updates that don't trigger the write barrier, but still look like a false reference.

// Thus, when we get here, we'll see a false reference that isn't marked.

// Since this situation is hard to detect, just don't verify mark for write barrier blocks.

// We could fix this if we had object layout info.

if (!this->IsLeafBlock()

#ifdef RECYCLER_WRITE_BARRIER

&& !this->IsWithBarrier()

#endif

)

{

if ((ObjectInfo(objectIndex) & LeafBit) == 0)

{

char * objectAddress = this->address + objectIndex * objectSize;

for (uint i = 0; i < objectWordCount; i++)

{

void* target = *(void**) objectAddress;

recycler->VerifyMark(target);

objectAddress += sizeof(void *);

}

}

}

}

}

NEXT_BITSET_IN_FIXEDBV;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::VerifyMark(void * objectAddress)

{

// Because we mark through new object, we might have a false reference

// somewhere that we have scanned before this new block is allocated

// so the object will not be marked even though it looks like a reference

// Can't verify when the block is new

if (this->heapBucket->GetRecycler()->heapBlockMap.IsAddressInNewChunk(objectAddress))

{

return;

}

ushort bitIndex = GetAddressBitIndex(objectAddress);

#if DBG

Assert(this->GetMarkedBitVector()->Test(bitIndex));

#else

if (!this->GetMarkedBitVector()->Test(bitIndex))

{

DebugBreak();

}

#endif

}

#endif

#ifdef RECYCLER_STRESS

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::InduceFalsePositive(Recycler * recycler)

{

// Induce a false positive mark by marking the first object on the free list, if any.

// Note that if the block is in the allocator, freeObjectList is not up to date.

// So we may be marking an already-allocated block, but that's okay --

// we call TryMark so that normal processing (including tracked object processing, etc)

// will occur just as if we had a false reference to this object previously.

void * falsePositive = this->freeObjectList;

if (falsePositive != nullptr)

{

recycler->TryMarkNonInterior(falsePositive, nullptr);

}

}

#endif

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::ClearAllAllocBytes()

{

#if ENABLE_PARTIAL_GC

this->oldFreeCount = this->lastFreeCount = this->freeCount;

#else

this->lastFreeCount = this->freeCount;

#endif

}

#if ENABLE_PARTIAL_GC

template <class TBlockAttributes>

bool

SmallHeapBlockT<TBlockAttributes>::DoPartialReusePage(RecyclerSweep const& recyclerSweep, uint& expectFreeByteCount)

{

// Partial GC page reuse heuristic

Assert(recyclerSweep.InPartialCollectMode());

expectFreeByteCount = GetExpectedFreeBytes();

// PartialCollectSmallHeapBlockReuseMinFreeBytes is calculated by dwPageSize* efficacy. If efficacy is

// high (== 1), and dwPageSize % objectSize != 0, all the pages in the bucket will be partial, and that

// could increase in thread sweep time.

// OTOH, if the object size is really large, the calculation below will reduce the chance for a page to be

// partial. we might need to watch out for that.

return (expectFreeByteCount + objectSize >= recyclerSweep.GetPartialCollectSmallHeapBlockReuseMinFreeBytes());

}

#if DBG

// do debug assert for partial block that we are not going to sweep

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::SweepVerifyPartialBlock(Recycler * recycler)

{

Assert(!this->IsLeafBlock());

// nothing in the partialHeapBlockList is sweepable

Assert(GetExpectedSweepObjectCount() == 0);

}

#endif

template <class TBlockAttributes>

uint

SmallHeapBlockT<TBlockAttributes>::GetAndClearUnaccountedAllocBytes()

{

Assert(this->lastFreeCount >= this->freeCount);

const ushort currentFreeCount = this->freeCount;

uint unaccountedAllocBytes = (this->lastFreeCount - currentFreeCount) * this->objectSize;

this->lastFreeCount = currentFreeCount;

return unaccountedAllocBytes;

}

template <class TBlockAttributes>

void

SmallHeapBlockT<TBlockAttributes>::AdjustPartialUncollectedAllocBytes(RecyclerSweep& recyclerSweep, uint const expectSweepCount)

{

const uint allObjectCount = this->objectCount;

const ushort currentFreeCount = this->freeCount;

Assert(this->lastFreeCount == currentFreeCount);

uint newAllocatedCount = this->oldFreeCount - currentFreeCount;

this->oldFreeCount = currentFreeCount;

uint newObjectExpectSweepCount = expectSweepCount;

#if ENABLE_CONCURRENT_GC

if (expectSweepCount != 0 && !recyclerSweep.InPartialCollect())

{

// We don't know which objects that we are going sweep are old and which object are new

// So just assume one way or the other by the amount of old vs. new object in the block

const uint allocatedObjectCount = allObjectCount - currentFreeCount;

Assert(allocatedObjectCount >= newAllocatedCount);

const uint oldObjectCount = allocatedObjectCount - newAllocatedCount;

if (oldObjectCount < newAllocatedCount)

{

// count all of the swept object as new, but don't exceed the amount we allocated

if (newObjectExpectSweepCount > newAllocatedCount)

{

newObjectExpectSweepCount = newAllocatedCount;

}

}

else

{

// count all of the swept object as old

newObjectExpectSweepCount = 0;

}

}

#endif

// The page can be old, or it is full (where we set lastFreeCount to 0)

// Otherwise, the newly allocated count must be bigger then the expect sweep count

Assert(newAllocatedCount >= newObjectExpectSweepCount);

Assert(this->lastUncollectedAllocBytes >= newObjectExpectSweepCount * this->objectSize);

recyclerSweep.SubtractSweepNewObjectAllocBytes(newObjectExpectSweepCount * this->objectSize);

}

#endif

template <class TBlockAttributes>

uint

SmallHeapBlockT<TBlockAttributes>::GetMarkCountForSweep()

{

Assert(IsFreeBitsValid());

// Make a local copy of mark bits, so we don't modify the actual mark bits.

SmallHeapBlockBitVector temp;

temp.Copy(this->GetMarkedBitVector());

// Remove any invalid bits that may have been set

temp.Minus(this->GetInvalidBitVector());

// Remove the mark bit for things that are still free

if (this->freeCount != 0)

{

temp.Minus(this->GetFreeBitVector());

}

return temp.Count();

}

template <class TBlockAttributes>

SweepState

SmallHeapBlockT<TBlockAttributes>::Sweep(RecyclerSweep& recyclerSweep, bool queuePendingSweep, bool allocable, ushort finalizeCount, bool hasPendingDispose)

{