berkus
/
essence-os
mirror of https://gitlab.com/nakst/essence
1
0
Fork 0
Code Releases Activity
essence-os/kernel/memory.cpp

2271 lines
73 KiB
C++
Raw Permalink Blame History

// This file is part of the Essence operating system.
// It is released under the terms of the MIT license -- see LICENSE.md.
// Written by: nakst.
// TODO Soft page faults.
// TODO Paging file.
// TODO Large pages.
// TODO Locking memory.
// TODO No execute permissions.
// TODO NUMA?
// TODO Cache coloring.
#define GUARD_PAGES
#ifndef IMPLEMENTATION
// A contiguous range of addresses in an MMSpace.
struct MMRegion {
uintptr_t baseAddress;
size_t pageCount;
uint32_t flags;
struct {
union {
struct {
uintptr_t offset;
} physical;
struct {
struct MMSharedRegion *region;
uintptr_t offset;
} shared;
struct {
struct FSFile *node;
EsFileOffset offset;
size_t zeroedBytes;
uint64_t fileHandleFlags;
} file;
struct {
RangeSet commit; // TODO Currently guarded by MMSpace::reserveMutex, maybe give it its own mutex?
size_t commitPageCount;
MMRegion *guardBefore, *guardAfter;
} normal;
};
KWriterLock pin; // Take SHARED when using the region in a system call. Take EXCLUSIVE to free the region.
KMutex mapMutex; // Access the page tables for a specific region, e.g. to atomically check if a mapping exists and if not, create it.
} data;
union {
struct {
AVLItem<MMRegion> itemBase;
union {
AVLItem<MMRegion> itemSize;
LinkedItem<MMRegion> itemNonGuard;
};
};
struct {
bool used;
} core;
};
};
// A virtual address space.
// One per process.
struct MMSpace {
MMArchVAS data; // Architecture specific data.
AVLTree<MMRegion> // Key =
freeRegionsBase, // Base address
freeRegionsSize, // Page count
usedRegions; // Base address
LinkedList<MMRegion> usedRegionsNonGuard;
KMutex reserveMutex; // Acquire to Access the region trees.
volatile int32_t referenceCount; // One per CPU using the space, and +1 while the process is alive.
// We don't bother tracking for kernelMMSpace.
bool user; // Regions in the space may be accessed from userspace.
uint64_t commit; // An *approximate* commit in pages. TODO Better memory usage tracking.
uint64_t reserve; // The number of reserved pages.
KAsyncTask removeAsyncTask; // The asynchronous task for deallocating the memory space once it's no longer in use.
};
// A physical page of memory.
struct MMPageFrame {
volatile enum : uint8_t {
// Frames that can't be used.
UNUSABLE, // Not connected to RAM.
BAD, // Connected to RAM with errors. TODO
// Frames that aren't referenced.
ZEROED, // Cleared with zeros.
FREE, // Contains whatever data is had when it was freed.
// Frames that are referenced by an invalid [shared] page mapping.
// In shared regions, each invalid page mapping points to the shared page mapping.
// This means only one variable must be updated to reuse the frame.
STANDBY, // Data has been written to page file or backing file.
// Frames that are referenced by one or more valid page mappings.
ACTIVE,
} state;
volatile uint8_t flags;
// The reference to this frame in a CCCachedSection.
volatile uintptr_t *cacheReference;
union {
struct {
// For STANDBY, MODIFIED, UPDATED, ZEROED and FREE.
// The frame's position in a list.
volatile uintptr_t next, *previous;
} list;
struct {
// For ACTIVE.
// For file pages, this tracks how many valid page table entries point to this frame.
volatile uintptr_t references;
} active;
};
};
// A shared memory region.
struct MMSharedRegion {
size_t sizeBytes;
volatile size_t handles;
KMutex mutex;
void *data;
};
// An object pool, for fast allocation and deallocation of objects of constant size.
// (There is no guarantee that the objects will be contiguous in memory.)
struct Pool {
void *Add(size_t elementSize); // Aligned to the size of a pointer.
void Remove(void *element);
size_t elementSize;
void *cache[POOL_CACHE_COUNT];
size_t cacheEntries;
KMutex mutex;
};
// Physical memory manager state.
struct PMM {
MMPageFrame *pageFrames;
bool pageFrameDatabaseInitialised;
uintptr_t pageFrameDatabaseCount;
uintptr_t firstFreePage;
uintptr_t firstZeroedPage;
uintptr_t firstStandbyPage, lastStandbyPage;
Bitset freeOrZeroedPageBitset; // Used for allocating large pages.
uintptr_t countZeroedPages, countFreePages, countStandbyPages, countActivePages;
#define MM_REMAINING_COMMIT() (pmm.commitLimit - pmm.commitPageable - pmm.commitFixed)
int64_t commitFixed, commitPageable,
commitFixedLimit, commitLimit;
// Acquire to:
KMutex commitMutex, // (Un)commit pages.
pageFrameMutex; // Allocate or free pages.
KMutex pmManipulationLock;
KSpinlock pmManipulationProcessorLock;
void *pmManipulationRegion;
Thread *zeroPageThread;
KEvent zeroPageEvent;
LinkedList<MMObjectCache> objectCacheList;
KMutex objectCacheListMutex;
// Events for when the number of available pages is low.
#define MM_AVAILABLE_PAGES() (pmm.countZeroedPages + pmm.countFreePages + pmm.countStandbyPages)
KEvent availableCritical, availableLow;
KEvent availableNotCritical;
// Event for when the object cache should be trimmed.
#define MM_OBJECT_CACHE_SHOULD_TRIM() (pmm.approximateTotalObjectCacheBytes / K_PAGE_SIZE > MM_OBJECT_CACHE_PAGES_MAXIMUM())
uintptr_t approximateTotalObjectCacheBytes;
KEvent trimObjectCaches;
// These variables will be cleared if the object they point to is removed.
// See MMUnreserve and Scheduler::RemoveProcess.
struct Process *nextProcessToBalance;
MMRegion *nextRegionToBalance;
uintptr_t balanceResumePosition;
};
// See MMPhysicalAllocate.
alignas(K_PAGE_SIZE) uint8_t earlyZeroBuffer[K_PAGE_SIZE];
// Constants.
// MMArchMapPage.
#define MM_MAP_PAGE_NOT_CACHEABLE (1 << 0)
#define MM_MAP_PAGE_USER (1 << 1)
#define MM_MAP_PAGE_OVERWRITE (1 << 2)
#define MM_MAP_PAGE_COMMIT_TABLES_NOW (1 << 3)
#define MM_MAP_PAGE_READ_ONLY (1 << 4)
#define MM_MAP_PAGE_COPIED (1 << 5)
#define MM_MAP_PAGE_NO_NEW_TABLES (1 << 6)
#define MM_MAP_PAGE_FRAME_LOCK_ACQUIRED (1 << 7)
#define MM_MAP_PAGE_WRITE_COMBINING (1 << 8)
#define MM_MAP_PAGE_IGNORE_IF_MAPPED (1 << 9)
// MMArchUnmapPages.
#define MM_UNMAP_PAGES_FREE (1 << 0)
#define MM_UNMAP_PAGES_FREE_COPIED (1 << 1)
#define MM_UNMAP_PAGES_BALANCE_FILE (1 << 2)
// MMPhysicalAllocate.
// --> Moved to module.h.
// MMHandlePageFault.
#define MM_HANDLE_PAGE_FAULT_WRITE (1 << 0)
#define MM_HANDLE_PAGE_FAULT_LOCK_ACQUIRED (1 << 1)
#define MM_HANDLE_PAGE_FAULT_FOR_SUPERVISOR (1 << 2)
// MMStandardAllocate - flags passed through into MMReserve.
// --> Moved to module.h.
// MMReserve - region types.
#define MM_REGION_PHYSICAL (0x0100) // The region is mapped to device memory.
#define MM_REGION_NORMAL (0x0200) // A normal region.
#define MM_REGION_SHARED (0x0400) // The region's contents is shared via a MMSharedRegion.
#define MM_REGION_GUARD (0x0800) // A guard region, to make sure we don't accidentally go into other regions.
#define MM_REGION_CACHE (0x1000) // Used for the file cache.
#define MM_REGION_FILE (0x2000) // A mapped file.
#define MM_SHARED_ENTRY_PRESENT (1)
// Forward declarations.
bool MMHandlePageFault(MMSpace *space, uintptr_t address, unsigned flags);
bool MMUnmapFilePage(uintptr_t frameNumber); // Returns true if the page became inactive.
void MMSharedDestroyRegion(MMSharedRegion *region);
// Public memory manager functions.
void *MMMapPhysical(MMSpace *space, uintptr_t address, size_t bytes, uint64_t caching);
void *MMStandardAllocate(MMSpace *space, size_t bytes, uint32_t flags, void *baseAddress, bool commitAll);
bool MMFree(MMSpace *space, void *address, size_t expectedSize, bool userOnly);
MMRegion *MMReserve(MMSpace *space, size_t bytes, unsigned flags, uintptr_t forcedAddress = 0, bool generateGuardPages = false);
bool MMCommit(uint64_t bytes, bool fixed);
void MMDecommit(uint64_t bytes, bool fixed);
bool MMCommitRange(MMSpace *space, MMRegion *region, uintptr_t offset, size_t pages);
bool MMDecommitRange(MMSpace *space, MMRegion *region, uintptr_t offset, size_t pages);
uintptr_t MMPhysicalAllocate(unsigned flags, uintptr_t count, uintptr_t align, uintptr_t below);
void MMInitialise();
MMSharedRegion *MMSharedCreateRegion(size_t sizeBytes, bool fixed, uintptr_t below);
MMRegion *MMFindAndPinRegion(MMSpace *space, uintptr_t address, uintptr_t size);
void MMUnpinRegion(MMSpace *space, MMRegion *region);
void MMSpaceDestroy(MMSpace *space);
bool MMSpaceInitialise(MMSpace *space);
void MMPhysicalFree(uintptr_t page, bool mutexAlreadyAcquired, size_t count);
void MMUnreserve(MMSpace *space, MMRegion *remove, bool unmapPages, bool guardRegion = false);
MMRegion *MMFindRegion(MMSpace *space, uintptr_t address);
void *MMMapFile(MMSpace *space, struct FSFile *node, EsFileOffset offset, size_t bytes,
int protection, void *baseAddress, size_t zeroedBytes = 0, uint32_t additionalFlags = ES_FLAGS_DEFAULT);
// Physical memory manipulation.
void PMZero(uintptr_t *pages, size_t pageCount,
bool contiguous /* True if `pages[0]` contains the base address and all other are contiguous, false if `pages` contains an array of physical addresses. */);
void PMZeroPartial(uintptr_t page, uintptr_t start, uintptr_t end);
void PMCheckZeroed(uintptr_t page);
void PMCopy(uintptr_t page, void *source, size_t pageCount);
#endif
#ifdef IMPLEMENTATION
// Globals.
MMSpace _kernelMMSpace, _coreMMSpace;
PMM pmm;
MMRegion *mmCoreRegions = (MMRegion *) MM_CORE_REGIONS_START;
size_t mmCoreRegionCount, mmCoreRegionArrayCommit;
MMSharedRegion *mmGlobalDataRegion, *mmAPITableRegion;
GlobalData *mmGlobalData; // Shared with all processes.
// Code!
void MMUpdateAvailablePageCount(bool increase) {
if (MM_AVAILABLE_PAGES() >= MM_CRITICAL_AVAILABLE_PAGES_THRESHOLD) {
KEventSet(&pmm.availableNotCritical, true);
KEventReset(&pmm.availableCritical);
} else {
KEventReset(&pmm.availableNotCritical);
KEventSet(&pmm.availableCritical, true);
if (!increase) {
KernelLog(LOG_ERROR, "Memory", "critical page limit hit",
"Critical number of available pages remain: %d (%dKB).\n", MM_AVAILABLE_PAGES(), MM_AVAILABLE_PAGES() * K_PAGE_SIZE / 1024);
}
}
if (MM_AVAILABLE_PAGES() >= MM_LOW_AVAILABLE_PAGES_THRESHOLD) {
KEventReset(&pmm.availableLow);
} else {
KEventSet(&pmm.availableLow, true);
}
}
void MMPhysicalInsertZeroedPage(uintptr_t page) {
if (GetCurrentThread() != pmm.zeroPageThread) {
KernelPanic("MMPhysicalInsertZeroedPage - Inserting a zeroed page not on the MMZeroPageThread.\n");
}
MMPageFrame *frame = pmm.pageFrames + page;
frame->state = MMPageFrame::ZEROED;
{
frame->list.next = pmm.firstZeroedPage;
frame->list.previous = &pmm.firstZeroedPage;
if (pmm.firstZeroedPage) pmm.pageFrames[pmm.firstZeroedPage].list.previous = &frame->list.next;
pmm.firstZeroedPage = page;
}
pmm.countZeroedPages++;
pmm.freeOrZeroedPageBitset.Put(page);
MMUpdateAvailablePageCount(true);
}
void MMPhysicalInsertFreePagesStart() {
}
void MMPhysicalInsertFreePagesNext(uintptr_t page) {
MMPageFrame *frame = pmm.pageFrames + page;
frame->state = MMPageFrame::FREE;
{
frame->list.next = pmm.firstFreePage;
frame->list.previous = &pmm.firstFreePage;
if (pmm.firstFreePage) pmm.pageFrames[pmm.firstFreePage].list.previous = &frame->list.next;
pmm.firstFreePage = page;
}
pmm.freeOrZeroedPageBitset.Put(page);
pmm.countFreePages++;
}
void MMPhysicalInsertFreePagesEnd() {
if (pmm.countFreePages > MM_ZERO_PAGE_THRESHOLD) {
KEventSet(&pmm.zeroPageEvent, true);
}
MMUpdateAvailablePageCount(true);
}
void MMPhysicalActivatePages(uintptr_t pages, uintptr_t count, unsigned flags) {
(void) flags;
KMutexAssertLocked(&pmm.pageFrameMutex);
for (uintptr_t i = 0; i < count; i++) {
MMPageFrame *frame = pmm.pageFrames + pages + i;
if (frame->state == MMPageFrame::FREE) {
pmm.countFreePages--;
} else if (frame->state == MMPageFrame::ZEROED) {
pmm.countZeroedPages--;
} else if (frame->state == MMPageFrame::STANDBY) {
pmm.countStandbyPages--;
if (pmm.lastStandbyPage == pages + i) {
if (frame->list.previous == &pmm.firstStandbyPage) {
// There are no more pages in the list.
pmm.lastStandbyPage = 0;
} else {
pmm.lastStandbyPage = ((uintptr_t) frame->list.previous - (uintptr_t) pmm.pageFrames) / sizeof(MMPageFrame);
}
}
} else {
KernelPanic("MMPhysicalActivatePages - Corrupt page frame database (4).\n");
}
// Unlink the frame from its list.
*frame->list.previous = frame->list.next;
if (frame->list.next) pmm.pageFrames[frame->list.next].list.previous = frame->list.previous;
EsMemoryZero(frame, sizeof(MMPageFrame));
frame->state = MMPageFrame::ACTIVE;
}
pmm.countActivePages += count;
MMUpdateAvailablePageCount(false);
}
uintptr_t MMPhysicalAllocate(unsigned flags, uintptr_t count, uintptr_t align, uintptr_t below) {
bool mutexAlreadyAcquired = flags & MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED;
if (!mutexAlreadyAcquired) KMutexAcquire(&pmm.pageFrameMutex);
else KMutexAssertLocked(&pmm.pageFrameMutex);
EsDefer(if (!mutexAlreadyAcquired) KMutexRelease(&pmm.pageFrameMutex););
intptr_t commitNow = count * K_PAGE_SIZE;
if (flags & MM_PHYSICAL_ALLOCATE_COMMIT_NOW) {
if (!MMCommit(commitNow, true)) return 0;
} else commitNow = 0;
bool simple = count == 1 && align == 1 && below == 0;
if (!pmm.pageFrameDatabaseInitialised) {
// Early page allocation before the page frame database is initialised.
if (!simple) {
KernelPanic("MMPhysicalAllocate - Non-simple allocation before initialisation of the page frame database.\n");
}
uintptr_t page = MMArchEarlyAllocatePage();
if (flags & MM_PHYSICAL_ALLOCATE_ZEROED) {
// TODO Hack!
MMArchMapPage(coreMMSpace, page, (uintptr_t) earlyZeroBuffer,
MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES | MM_MAP_PAGE_FRAME_LOCK_ACQUIRED);
EsMemoryZero(earlyZeroBuffer, K_PAGE_SIZE);
}
return page;
} else if (!simple) {
// Slow path.
// TODO Use standby pages.
uintptr_t pages = pmm.freeOrZeroedPageBitset.Get(count, align, below);
if (pages == (uintptr_t) -1) goto fail;
MMPhysicalActivatePages(pages, count, flags);
uintptr_t address = pages << K_PAGE_BITS;
if (flags & MM_PHYSICAL_ALLOCATE_ZEROED) PMZero(&address, count, true);
return address;
} else {
uintptr_t page = 0;
bool notZeroed = false;
if (!page) page = pmm.firstZeroedPage;
if (!page) page = pmm.firstFreePage, notZeroed = true;
if (!page) page = pmm.lastStandbyPage, notZeroed = true;
if (!page) goto fail;
MMPageFrame *frame = pmm.pageFrames + page;
if (frame->state == MMPageFrame::ACTIVE) {
KernelPanic("MMPhysicalAllocate - Corrupt page frame database (2).\n");
}
if (frame->state == MMPageFrame::STANDBY) {
// EsPrint("Clear RT %x\n", frame);
if (*frame->cacheReference != ((page << K_PAGE_BITS) | MM_SHARED_ENTRY_PRESENT)) {
KernelPanic("MMPhysicalAllocate - Corrupt shared reference back pointer in frame %x.\n", frame);
}
// Clear the entry in the CCCachedSection that referenced this standby frame.
*frame->cacheReference = 0;
// TODO If the CCCachedSection is empty, remove it from its CCSpace.
} else {
pmm.freeOrZeroedPageBitset.Take(page);
}
MMPhysicalActivatePages(page, 1, flags);
// EsPrint("PAGE FRAME ALLOCATE: %x\n", page << K_PAGE_BITS);
uintptr_t address = page << K_PAGE_BITS;
if (notZeroed && (flags & MM_PHYSICAL_ALLOCATE_ZEROED)) PMZero(&address, 1, false);
// if (!notZeroed) PMCheckZeroed(address);
return address;
}
fail:;
if (!(flags & MM_PHYSICAL_ALLOCATE_CAN_FAIL)) {
EsPrint("Out of memory. Committed %d/%d fixed and %d pageable out of a maximum %d.\n", pmm.commitFixed, pmm.commitFixedLimit, pmm.commitPageable, pmm.commitLimit);
KernelPanic("MMPhysicalAllocate - Out of memory.\n");
}
MMDecommit(commitNow, true);
return 0;
}
void MMPhysicalFree(uintptr_t page, bool mutexAlreadyAcquired, size_t count) {
if (!page) KernelPanic("MMPhysicalFree - Invalid page.\n");
if (mutexAlreadyAcquired) KMutexAssertLocked(&pmm.pageFrameMutex);
else KMutexAcquire(&pmm.pageFrameMutex);
if (!pmm.pageFrameDatabaseInitialised) KernelPanic("MMPhysicalFree - PMM not yet initialised.\n");
page >>= K_PAGE_BITS;
MMPhysicalInsertFreePagesStart();
for (uintptr_t i = 0; i < count; i++, page++) {
MMPageFrame *frame = pmm.pageFrames + page;
if (frame->state == MMPageFrame::FREE) {
KernelPanic("MMPhysicalFree - Attempting to free a FREE page.\n");
}
if (pmm.commitFixedLimit) {
pmm.countActivePages--;
}
MMPhysicalInsertFreePagesNext(page);
}
MMPhysicalInsertFreePagesEnd();
if (!mutexAlreadyAcquired) KMutexRelease(&pmm.pageFrameMutex);
}
void MMCheckUnusable(uintptr_t physicalStart, size_t bytes) {
for (uintptr_t i = physicalStart / K_PAGE_SIZE; i < (physicalStart + bytes + K_PAGE_SIZE - 1) / K_PAGE_SIZE
&& i < pmm.pageFrameDatabaseCount; i++) {
if (pmm.pageFrames[i].state != MMPageFrame::UNUSABLE) {
KernelPanic("MMCheckUnusable - Page frame at address %x should be unusable.\n", i * K_PAGE_SIZE);
}
}
}
MMRegion *MMFindRegion(MMSpace *space, uintptr_t address) {
KMutexAssertLocked(&space->reserveMutex);
if (space == coreMMSpace) {
for (uintptr_t i = 0; i < mmCoreRegionCount; i++) {
MMRegion *region = mmCoreRegions + i;
if (region->core.used && region->baseAddress <= address
&& region->baseAddress + region->pageCount * K_PAGE_SIZE > address) {
return region;
}
}
} else {
AVLItem<MMRegion> *item = TreeFind(&space->usedRegions, MakeShortKey(address), TREE_SEARCH_LARGEST_BELOW_OR_EQUAL);
if (!item) return nullptr;
MMRegion *region = item->thisItem;
if (region->baseAddress > address) KernelPanic("MMFindRegion - Broken usedRegions tree.\n");
if (region->baseAddress + region->pageCount * K_PAGE_SIZE <= address) return nullptr;
return region;
}
return nullptr;
}
uintptr_t MMSharedLookupPage(MMSharedRegion *region, uintptr_t pageIndex) {
KMutexAcquire(&region->mutex);
if (pageIndex >= region->sizeBytes >> K_PAGE_BITS) KernelPanic("MMSharedLookupPage - Page %d out of range in region %x.\n", pageIndex, region);
uintptr_t entry = ((uintptr_t *) region->data)[pageIndex];
KMutexRelease(&region->mutex);
return entry;
}
bool MMHandlePageFault(MMSpace *space, uintptr_t address, unsigned faultFlags) {
// EsPrint("HandlePageFault: %x/%x/%x\n", space, address, faultFlags);
address &= ~(K_PAGE_SIZE - 1);
bool lockAcquired = faultFlags & MM_HANDLE_PAGE_FAULT_LOCK_ACQUIRED;
MMRegion *region;
if (!lockAcquired && MM_AVAILABLE_PAGES() < MM_CRITICAL_AVAILABLE_PAGES_THRESHOLD && GetCurrentThread() && !GetCurrentThread()->isPageGenerator) {
KernelLog(LOG_ERROR, "Memory", "waiting for non-critical state", "Page fault on non-generator thread, waiting for more available pages.\n");
KEventWait(&pmm.availableNotCritical);
}
{
if (!lockAcquired) KMutexAcquire(&space->reserveMutex);
else KMutexAssertLocked(&space->reserveMutex);
EsDefer(if (!lockAcquired) KMutexRelease(&space->reserveMutex));
// Find the region, and pin it (so it can't be freed).
region = MMFindRegion(space, address);
if (!region) return false;
if (!KWriterLockTake(&region->data.pin, K_LOCK_SHARED, true /* poll */)) return false;
}
EsDefer(KWriterLockReturn(&region->data.pin, K_LOCK_SHARED));
KMutexAcquire(&region->data.mapMutex);
EsDefer(KMutexRelease(&region->data.mapMutex));
if (MMArchTranslateAddress(space, address, faultFlags & MM_HANDLE_PAGE_FAULT_WRITE)) {
// Spurious page fault.
return true;
}
bool copyOnWrite = false, markModified = false;
if (faultFlags & MM_HANDLE_PAGE_FAULT_WRITE) {
if (region->flags & MM_REGION_COPY_ON_WRITE) {
// This is copy-on-write page that needs to be copied.
copyOnWrite = true;
} else if (region->flags & MM_REGION_READ_ONLY) {
// The page was read-only.
KernelLog(LOG_ERROR, "Memory", "read only page fault", "MMHandlePageFault - Page was read only.\n");
return false;
} else {
// We mapped the page as read-only so we could track whether it has been written to.
// It has now been written to.
markModified = true;
}
}
uintptr_t offsetIntoRegion = address - region->baseAddress;
uint64_t needZeroPages = 0;
bool zeroPage = true;
if (space->user) {
needZeroPages = MM_PHYSICAL_ALLOCATE_ZEROED;
zeroPage = false;
}
unsigned flags = ES_FLAGS_DEFAULT;
if (space->user) flags |= MM_MAP_PAGE_USER;
if (region->flags & MM_REGION_NOT_CACHEABLE) flags |= MM_MAP_PAGE_NOT_CACHEABLE;
if (region->flags & MM_REGION_WRITE_COMBINING) flags |= MM_MAP_PAGE_WRITE_COMBINING;
if (!markModified && !(region->flags & MM_REGION_FIXED) && (region->flags & MM_REGION_FILE)) flags |= MM_MAP_PAGE_READ_ONLY;
if (region->flags & MM_REGION_PHYSICAL) {
MMArchMapPage(space, region->data.physical.offset + address - region->baseAddress, address, flags);
return true;
} else if (region->flags & MM_REGION_SHARED) {
MMSharedRegion *sharedRegion = region->data.shared.region;
if (!sharedRegion->handles) {
KernelPanic("MMHandlePageFault - Shared region has no handles.\n");
}
KMutexAcquire(&sharedRegion->mutex);
uintptr_t offset = address - region->baseAddress + region->data.shared.offset;
if (offset >= sharedRegion->sizeBytes) {
KMutexRelease(&sharedRegion->mutex);
KernelLog(LOG_ERROR, "Memory", "outside shared size", "MMHandlePageFault - Attempting to access shared memory past end of region.\n");
return false;
}
uintptr_t *entry = (uintptr_t *) sharedRegion->data + (offset / K_PAGE_SIZE);
if (*entry & MM_SHARED_ENTRY_PRESENT) zeroPage = false;
else *entry = MMPhysicalAllocate(needZeroPages) | MM_SHARED_ENTRY_PRESENT;
MMArchMapPage(space, *entry & ~(K_PAGE_SIZE - 1), address, flags);
if (zeroPage) EsMemoryZero((void *) address, K_PAGE_SIZE);
KMutexRelease(&sharedRegion->mutex);
return true;
} else if (region->flags & MM_REGION_FILE) {
if (address >= region->baseAddress + (region->pageCount << K_PAGE_BITS) - region->data.file.zeroedBytes) {
// EsPrint("%x:%d\n", address, needZeroPages);
MMArchMapPage(space, MMPhysicalAllocate(needZeroPages), address, (flags & ~MM_MAP_PAGE_READ_ONLY) | MM_MAP_PAGE_COPIED);
if (zeroPage) EsMemoryZero((void *) address, K_PAGE_SIZE);
return true;
}
if (copyOnWrite) {
doCopyOnWrite:;
uintptr_t existingTranslation = MMArchTranslateAddress(space, address);
if (existingTranslation) {
// We need to use PMCopy, because as soon as the page is mapped for the user,
// other threads can access it.
uintptr_t page = MMPhysicalAllocate(ES_FLAGS_DEFAULT);
PMCopy(page, (void *) address, 1);
MMArchUnmapPages(space, address, 1, MM_UNMAP_PAGES_BALANCE_FILE);
MMArchMapPage(space, page, address, (flags & ~MM_MAP_PAGE_READ_ONLY) | MM_MAP_PAGE_COPIED);
return true;
} else {
// EsPrint("Write to unmapped, %x.\n", address);
}
}
KMutexRelease(&region->data.mapMutex);
size_t pagesToRead = 16;
if (region->pageCount - (offsetIntoRegion + region->data.file.zeroedBytes) / K_PAGE_SIZE < pagesToRead) {
pagesToRead = region->pageCount - (offsetIntoRegion + region->data.file.zeroedBytes) / K_PAGE_SIZE;
}
// This shouldn't block, because mapped files cannot be resized.
KWriterLockTake(&region->data.file.node->resizeLock, K_LOCK_SHARED);
EsError error = CCSpaceAccess(&region->data.file.node->cache, (void *) address,
offsetIntoRegion + region->data.file.offset, pagesToRead * K_PAGE_SIZE,
CC_ACCESS_MAP, space, flags);
KWriterLockReturn(&region->data.file.node->resizeLock, K_LOCK_SHARED);
KMutexAcquire(&region->data.mapMutex);
if (error != ES_SUCCESS) {
KernelLog(LOG_ERROR, "Memory", "mapped file read error", "MMHandlePageFault - Could not read from file %x. Error: %d.\n",
region->data.file.node, error);
return false;
}
if (copyOnWrite) {
goto doCopyOnWrite;
}
return true;
} else if (region->flags & MM_REGION_NORMAL) {
if (!(region->flags & MM_REGION_NO_COMMIT_TRACKING)) {
if (!region->data.normal.commit.Contains(offsetIntoRegion >> K_PAGE_BITS)) {
KernelLog(LOG_ERROR, "Memory", "outside commit range", "MMHandlePageFault - Attempting to access uncommitted memory in reserved region.\n");
return false;
}
}
MMArchMapPage(space, MMPhysicalAllocate(needZeroPages), address, flags);
if (zeroPage) EsMemoryZero((void *) address, K_PAGE_SIZE);
return true;
} else if (region->flags & MM_REGION_GUARD) {
KernelLog(LOG_ERROR, "Memory", "guard page hit", "MMHandlePageFault - Guard page hit!\n");
return false;
} else {
KernelLog(LOG_ERROR, "Memory", "cannot fault in region", "MMHandlePageFault - Unsupported region type (flags: %x).\n", region->flags);
return false;
}
}
MMRegion *MMReserve(MMSpace *space, size_t bytes, unsigned flags, uintptr_t forcedAddress, bool generateGuardPages) {
// TODO Handling EsHeapAllocate failures.
MMRegion *outputRegion = nullptr;
size_t pagesNeeded = ((bytes + K_PAGE_SIZE - 1) & ~(K_PAGE_SIZE - 1)) / K_PAGE_SIZE;
if (!pagesNeeded) return nullptr;
KMutexAssertLocked(&space->reserveMutex);
if (space == coreMMSpace) {
if (mmCoreRegionCount == MM_CORE_REGIONS_COUNT) {
return nullptr;
}
if (forcedAddress) {
KernelPanic("MMReserve - Using a forced address in coreMMSpace.\n");
}
{
uintptr_t newRegionCount = mmCoreRegionCount + 1;
uintptr_t commitPagesNeeded = newRegionCount * sizeof(MMRegion) / K_PAGE_SIZE + 1;
while (mmCoreRegionArrayCommit < commitPagesNeeded) {
if (!MMCommit(K_PAGE_SIZE, true)) return nullptr;
mmCoreRegionArrayCommit++;
}
}
for (uintptr_t i = 0; i < mmCoreRegionCount; i++) {
MMRegion *region = mmCoreRegions + i;
if (!region->core.used && region->pageCount >= pagesNeeded) {
if (region->pageCount > pagesNeeded) {
MMRegion *split = mmCoreRegions + mmCoreRegionCount++;
EsMemoryCopy(split, region, sizeof(MMRegion));
split->baseAddress += pagesNeeded * K_PAGE_SIZE;
split->pageCount -= pagesNeeded;
}
region->core.used = true;
region->pageCount = pagesNeeded;
region->flags = flags;
EsMemoryZero(&region->data, sizeof(region->data));
outputRegion = region;
goto done;
}
}
} else if (forcedAddress) {
AVLItem<MMRegion> *item;
// EsPrint("reserve forced: %x\n", forcedAddress);
// Check for a collision.
item = TreeFind(&space->usedRegions, MakeShortKey(forcedAddress), TREE_SEARCH_EXACT);
if (item) return nullptr;
item = TreeFind(&space->usedRegions, MakeShortKey(forcedAddress), TREE_SEARCH_SMALLEST_ABOVE_OR_EQUAL);
if (item && item->thisItem->baseAddress < forcedAddress + pagesNeeded * K_PAGE_SIZE) return nullptr;
item = TreeFind(&space->usedRegions, MakeShortKey(forcedAddress + pagesNeeded * K_PAGE_SIZE - 1), TREE_SEARCH_LARGEST_BELOW_OR_EQUAL);
if (item && item->thisItem->baseAddress + item->thisItem->pageCount * K_PAGE_SIZE > forcedAddress) return nullptr;
item = TreeFind(&space->freeRegionsBase, MakeShortKey(forcedAddress), TREE_SEARCH_EXACT);
if (item) return nullptr;
item = TreeFind(&space->freeRegionsBase, MakeShortKey(forcedAddress), TREE_SEARCH_SMALLEST_ABOVE_OR_EQUAL);
if (item && item->thisItem->baseAddress < forcedAddress + pagesNeeded * K_PAGE_SIZE) return nullptr;
item = TreeFind(&space->freeRegionsBase, MakeShortKey(forcedAddress + pagesNeeded * K_PAGE_SIZE - 1), TREE_SEARCH_LARGEST_BELOW_OR_EQUAL);
if (item && item->thisItem->baseAddress + item->thisItem->pageCount * K_PAGE_SIZE > forcedAddress) return nullptr;
// EsPrint("(no collisions)\n");
MMRegion *region = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
region->baseAddress = forcedAddress;
region->pageCount = pagesNeeded;
region->flags = flags;
TreeInsert(&space->usedRegions, &region->itemBase, region, MakeShortKey(region->baseAddress));
EsMemoryZero(&region->data, sizeof(region->data));
outputRegion = region;
} else {
#ifdef GUARD_PAGES
size_t guardPagesNeeded = generateGuardPages ? 2 : 0;
#else
size_t guardPagesNeeded = 0;
#endif
AVLItem<MMRegion> *item = TreeFind(&space->freeRegionsSize, MakeShortKey(pagesNeeded + guardPagesNeeded), TREE_SEARCH_SMALLEST_ABOVE_OR_EQUAL);
if (!item) {
goto done;
}
MMRegion *region = item->thisItem;
TreeRemove(&space->freeRegionsBase, &region->itemBase);
TreeRemove(&space->freeRegionsSize, &region->itemSize);
if (region->pageCount > pagesNeeded + guardPagesNeeded) {
MMRegion *split = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
EsMemoryCopy(split, region, sizeof(MMRegion));
split->baseAddress += (pagesNeeded + guardPagesNeeded) * K_PAGE_SIZE;
split->pageCount -= (pagesNeeded + guardPagesNeeded);
TreeInsert(&space->freeRegionsBase, &split->itemBase, split, MakeShortKey(split->baseAddress));
TreeInsert(&space->freeRegionsSize, &split->itemSize, split, MakeShortKey(split->pageCount), AVL_DUPLICATE_KEYS_ALLOW);
}
EsMemoryZero(&region->data, sizeof(region->data));
region->pageCount = pagesNeeded;
region->flags = flags;
if (guardPagesNeeded) {
MMRegion *guardBefore = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
MMRegion *guardAfter = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
EsMemoryCopy(guardBefore, region, sizeof(MMRegion));
EsMemoryCopy(guardAfter, region, sizeof(MMRegion));
guardAfter->baseAddress += K_PAGE_SIZE * (pagesNeeded + 1);
guardBefore->pageCount = guardAfter->pageCount = 1;
guardBefore->flags = guardAfter->flags = MM_REGION_GUARD;
region->baseAddress += K_PAGE_SIZE;
region->data.normal.guardBefore = guardBefore;
region->data.normal.guardAfter = guardAfter;
EsMemoryZero(&guardBefore->itemNonGuard, sizeof(guardBefore->itemNonGuard));
EsMemoryZero(&guardAfter->itemNonGuard, sizeof(guardAfter->itemNonGuard));
TreeInsert(&space->usedRegions, &guardBefore->itemBase, guardBefore, MakeShortKey(guardBefore->baseAddress));
TreeInsert(&space->usedRegions, &guardAfter ->itemBase, guardAfter, MakeShortKey(guardAfter->baseAddress));
#if 0
EsPrint("Guarded region: %x->%x/%x->%x/%x->%x\n", guardBefore->baseAddress, guardBefore->pageCount * K_PAGE_SIZE + guardBefore->baseAddress,
region->baseAddress, region->pageCount * K_PAGE_SIZE + region->baseAddress,
guardAfter->baseAddress, guardAfter->pageCount * K_PAGE_SIZE + guardAfter->baseAddress);
#endif
}
TreeInsert(&space->usedRegions, &region->itemBase, region, MakeShortKey(region->baseAddress));
outputRegion = region;
goto done;
}
done:;
// EsPrint("reserve: %x -> %x\n", address, (uintptr_t) address + pagesNeeded * K_PAGE_SIZE);
if (outputRegion) {
// We've now got an address range for the region.
// So we should commit the page tables that will be needed to map it.
if (!MMArchCommitPageTables(space, outputRegion)) {
// We couldn't commit the leading page tables.
// So we'll have to unreserve the region.
MMUnreserve(space, outputRegion, false);
return nullptr;
}
if (space != coreMMSpace) {
EsMemoryZero(&outputRegion->itemNonGuard, sizeof(outputRegion->itemNonGuard));
outputRegion->itemNonGuard.thisItem = outputRegion;
space->usedRegionsNonGuard.InsertEnd(&outputRegion->itemNonGuard);
}
space->reserve += pagesNeeded;
}
// EsPrint("Reserve: %x->%x\n", outputRegion->baseAddress, outputRegion->pageCount * K_PAGE_SIZE + outputRegion->baseAddress);
return outputRegion;
}
void MMUnreserve(MMSpace *space, MMRegion *remove, bool unmapPages, bool guardRegion) {
// EsPrint("unreserve: %x, %x, %d, %d\n", remove->baseAddress, remove->flags, unmapPages, guardRegion);
// EsDefer(EsPrint("unreserve complete\n"););
KMutexAssertLocked(&space->reserveMutex);
if (pmm.nextRegionToBalance == remove) {
// If the balance thread paused while balancing this region,
// switch to the next region.
pmm.nextRegionToBalance = remove->itemNonGuard.nextItem ? remove->itemNonGuard.nextItem->thisItem : nullptr;
pmm.balanceResumePosition = 0;
}
if (!remove) {
KernelPanic("MMUnreserve - Region to remove was null.\n");
}
if (remove->flags & MM_REGION_NORMAL) {
if (remove->data.normal.guardBefore) MMUnreserve(space, remove->data.normal.guardBefore, false, true);
if (remove->data.normal.guardAfter) MMUnreserve(space, remove->data.normal.guardAfter, false, true);
} else if ((remove->flags & MM_REGION_GUARD) && !guardRegion) {
// You can't free a guard region!
// TODO Error.
KernelLog(LOG_ERROR, "Memory", "attempt to unreserve guard page", "MMUnreserve - Attempting to unreserve a guard page.\n");
return;
}
if (remove->itemNonGuard.list && !guardRegion) {
// EsPrint("Remove item non guard...\n");
remove->itemNonGuard.RemoveFromList();
// EsPrint("Removed.\n");
}
if (unmapPages) {
MMArchUnmapPages(space, remove->baseAddress, remove->pageCount, ES_FLAGS_DEFAULT);
}
space->reserve += remove->pageCount;
if (space == coreMMSpace) {
remove->core.used = false;
intptr_t remove1 = -1, remove2 = -1;
for (uintptr_t i = 0; i < mmCoreRegionCount && (remove1 != -1 || remove2 != 1); i++) {
MMRegion *r = mmCoreRegions + i;
if (r->core.used) continue;
if (r == remove) continue;
if (r->baseAddress == remove->baseAddress + (remove->pageCount << K_PAGE_BITS)) {
remove->pageCount += r->pageCount;
remove1 = i;
} else if (remove->baseAddress == r->baseAddress + (r->pageCount << K_PAGE_BITS)) {
remove->pageCount += r->pageCount;
remove->baseAddress = r->baseAddress;
remove2 = i;
}
}
if (remove1 != -1) {
mmCoreRegions[remove1] = mmCoreRegions[--mmCoreRegionCount];
if ((uintptr_t) remove2 == mmCoreRegionCount) remove2 = remove1;
}
if (remove2 != -1) {
mmCoreRegions[remove2] = mmCoreRegions[--mmCoreRegionCount];
}
} else {
TreeRemove(&space->usedRegions, &remove->itemBase);
uintptr_t address = remove->baseAddress;
{
AVLItem<MMRegion> *before = TreeFind(&space->freeRegionsBase, MakeShortKey(address), TREE_SEARCH_LARGEST_BELOW_OR_EQUAL);
if (before && before->thisItem->baseAddress + before->thisItem->pageCount * K_PAGE_SIZE == remove->baseAddress) {
remove->baseAddress = before->thisItem->baseAddress;
remove->pageCount += before->thisItem->pageCount;
TreeRemove(&space->freeRegionsBase, before);
TreeRemove(&space->freeRegionsSize, &before->thisItem->itemSize);
EsHeapFree(before->thisItem, sizeof(MMRegion), K_CORE);
}
}
{
AVLItem<MMRegion> *after = TreeFind(&space->freeRegionsBase, MakeShortKey(address), TREE_SEARCH_SMALLEST_ABOVE_OR_EQUAL);
if (after && remove->baseAddress + remove->pageCount * K_PAGE_SIZE == after->thisItem->baseAddress) {
remove->pageCount += after->thisItem->pageCount;
TreeRemove(&space->freeRegionsBase, after);
TreeRemove(&space->freeRegionsSize, &after->thisItem->itemSize);
EsHeapFree(after->thisItem, sizeof(MMRegion), K_CORE);
}
}
TreeInsert(&space->freeRegionsBase, &remove->itemBase, remove, MakeShortKey(remove->baseAddress));
TreeInsert(&space->freeRegionsSize, &remove->itemSize, remove, MakeShortKey(remove->pageCount), AVL_DUPLICATE_KEYS_ALLOW);
}
}
void *MMMapFile(MMSpace *space, FSFile *node, EsFileOffset offset, size_t bytes, int protection, void *baseAddress, size_t zeroedBytes, uint32_t additionalFlags) {
if (protection != ES_MEMORY_MAP_OBJECT_READ_ONLY && protection != ES_MEMORY_MAP_OBJECT_COPY_ON_WRITE) {
return nullptr;
}
if (node->directoryEntry->type != ES_NODE_FILE) {
return nullptr;
}
MMRegion *region = nullptr;
uint64_t fileHandleFlags = ES_NODE_PREVENT_RESIZE
| (protection == ES_MEMORY_MAP_OBJECT_READ_WRITE ? ES_FILE_WRITE_SHARED : ES_FILE_READ_SHARED);
bool decommit = false;
// Register a handle to the node.
// If this successes, past this point we file cannot be resized.
if (!OpenHandleToObject(node, KERNEL_OBJECT_NODE, fileHandleFlags)) {
return nullptr;
}
// Acquire the file's cache mutex before the space's reserve mutex,
// so that we can handle page faults without locking the reserve mutex.
KMutexAcquire(&node->cache.cachedSectionsMutex);
EsDefer(KMutexRelease(&node->cache.cachedSectionsMutex));
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
// Check the parameters.
uintptr_t offsetIntoPage = offset & (K_PAGE_SIZE - 1);
if (node->directoryEntry->totalSize <= offset) {
goto fail;
}
if (offsetIntoPage) bytes += offset & (K_PAGE_SIZE - 1);
// Reserve the region.
region = MMReserve(space, bytes + zeroedBytes, MM_REGION_FILE | additionalFlags
| ((protection == ES_MEMORY_MAP_OBJECT_READ_ONLY || protection == ES_MEMORY_MAP_OBJECT_COPY_ON_WRITE) ? MM_REGION_READ_ONLY : 0)
| (protection == ES_MEMORY_MAP_OBJECT_COPY_ON_WRITE ? MM_REGION_COPY_ON_WRITE : 0),
baseAddress ? (uintptr_t) baseAddress - offsetIntoPage : 0);
if (!region) {
goto fail;
}
// Commit copy on write regions.
if (protection == ES_MEMORY_MAP_OBJECT_COPY_ON_WRITE) {
if (!MMCommit(bytes + zeroedBytes, false)) {
goto fail;
}
decommit = true;
}
// Initialise data.
region->data.file.node = node;
region->data.file.offset = offset - offsetIntoPage;
region->data.file.zeroedBytes = zeroedBytes;
region->data.file.fileHandleFlags = fileHandleFlags;
// Make sure the region is covered by MMCachedSections in the file.
// (This is why we needed the file's cache mutex.)
// We'll allocate in coreMMSpace as needed.
if (!CCSpaceCover(&node->cache, region->data.file.offset, region->data.file.offset + bytes)) {
goto fail;
}
return (uint8_t *) region->baseAddress + offsetIntoPage;
fail:;
if (region) MMUnreserve(space, region, false /* No pages have been mapped. */);
if (decommit) MMDecommit(bytes + zeroedBytes, false);
CloseHandleToObject(node, KERNEL_OBJECT_NODE, fileHandleFlags);
return nullptr;
}
void *MMMapShared(MMSpace *space, MMSharedRegion *sharedRegion, uintptr_t offset, size_t bytes, uint32_t additionalFlags, void *baseAddress) {
MMRegion *region;
OpenHandleToObject(sharedRegion, KERNEL_OBJECT_SHMEM);
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
if (offset & (K_PAGE_SIZE - 1)) bytes += offset & (K_PAGE_SIZE - 1);
if (sharedRegion->sizeBytes <= offset) goto fail;
if (sharedRegion->sizeBytes < offset + bytes) goto fail;
region = MMReserve(space, bytes, MM_REGION_SHARED | additionalFlags, (uintptr_t) baseAddress);
if (!region) goto fail;
if (!(region->flags & MM_REGION_SHARED)) KernelPanic("MMMapShared - Cannot commit into non-shared region.\n");
if (region->data.shared.region) KernelPanic("MMMapShared - A shared region has already been bound.\n");
region->data.shared.region = sharedRegion;
region->data.shared.offset = offset & ~(K_PAGE_SIZE - 1);
return (uint8_t *) region->baseAddress + (offset & (K_PAGE_SIZE - 1));
fail:;
CloseHandleToObject(sharedRegion, KERNEL_OBJECT_SHMEM);
return nullptr;
}
void MMDecommit(uint64_t bytes, bool fixed) {
// EsPrint("De-Commit %d %d\n", bytes, fixed);
if (bytes & (K_PAGE_SIZE - 1)) KernelPanic("MMDecommit - Expected multiple of K_PAGE_SIZE bytes.\n");
int64_t pagesNeeded = bytes / K_PAGE_SIZE;
KMutexAcquire(&pmm.commitMutex);
EsDefer(KMutexRelease(&pmm.commitMutex));
if (fixed) {
if (pmm.commitFixed < pagesNeeded) KernelPanic("MMDecommit - Decommitted too many pages.\n");
pmm.commitFixed -= pagesNeeded;
} else {
if (pmm.commitPageable < pagesNeeded) KernelPanic("MMDecommit - Decommitted too many pages.\n");
pmm.commitPageable -= pagesNeeded;
}
KernelLog(LOG_VERBOSE, "Memory", "decommit", "Decommit %D%z. Now at %D.\n", bytes, fixed ? ", fixed" : "", (pmm.commitFixed + pmm.commitPageable) << K_PAGE_BITS);
}
bool MMCommit(uint64_t bytes, bool fixed) {
if (bytes & (K_PAGE_SIZE - 1)) KernelPanic("MMCommit - Expected multiple of K_PAGE_SIZE bytes.\n");
int64_t pagesNeeded = bytes / K_PAGE_SIZE;
KMutexAcquire(&pmm.commitMutex);
EsDefer(KMutexRelease(&pmm.commitMutex));
if (pmm.commitLimit) {
if (fixed) {
if (pagesNeeded > pmm.commitFixedLimit - pmm.commitFixed) {
KernelLog(LOG_ERROR, "Memory", "failed fixed commit", "Failed fixed commit %d pages (currently %d/%d).\n",
pagesNeeded, pmm.commitFixed, pmm.commitFixedLimit);
return false;
}
if (MM_AVAILABLE_PAGES() - pagesNeeded < MM_CRITICAL_AVAILABLE_PAGES_THRESHOLD && !GetCurrentThread()->isPageGenerator) {
KernelLog(LOG_ERROR, "Memory", "failed fixed commit",
"Failed fixed commit %d as the available pages (%d, %d, %d) would cross the critical threshold, %d.\n.",
pagesNeeded, pmm.countZeroedPages, pmm.countFreePages, pmm.countStandbyPages, MM_CRITICAL_AVAILABLE_PAGES_THRESHOLD);
return false;
}
pmm.commitFixed += pagesNeeded;
} else {
if (pagesNeeded > MM_REMAINING_COMMIT() - (intptr_t) (GetCurrentThread()->isPageGenerator ? 0 : MM_CRITICAL_REMAINING_COMMIT_THRESHOLD)) {
KernelLog(LOG_ERROR, "Memory", "failed pageable commit",
"Failed pageable commit %d pages (currently %d/%d).\n",
pagesNeeded, pmm.commitPageable + pmm.commitFixed, pmm.commitLimit);
return false;
}
pmm.commitPageable += pagesNeeded;
}
if (MM_OBJECT_CACHE_SHOULD_TRIM()) {
KEventSet(&pmm.trimObjectCaches, true);
}
} else {
// We haven't started tracking commit counts yet.
}
KernelLog(LOG_VERBOSE, "Memory", "commit", "Commit %D%z. Now at %D.\n", bytes, fixed ? ", fixed" : "", (pmm.commitFixed + pmm.commitPageable) << K_PAGE_BITS);
return true;
}
bool MMCommitRange(MMSpace *space, MMRegion *region, uintptr_t pageOffset, size_t pageCount) {
KMutexAssertLocked(&space->reserveMutex);
if (region->flags & MM_REGION_NO_COMMIT_TRACKING) {
KernelPanic("MMCommitRange - Region does not support commit tracking.\n");
}
if (pageOffset >= region->pageCount || pageCount > region->pageCount - pageOffset) {
KernelPanic("MMCommitRange - Invalid region offset and page count.\n");
}
if (~region->flags & MM_REGION_NORMAL) {
KernelPanic("MMCommitRange - Cannot commit into non-normal region.\n");
}
intptr_t delta = 0;
region->data.normal.commit.Set(pageOffset, pageOffset + pageCount, &delta, false);
if (delta < 0) {
KernelPanic("MMCommitRange - Invalid delta calculation adding %x, %x to %x.\n", pageOffset, pageCount, region);
}
if (delta == 0) {
return true;
}
{
if (!MMCommit(delta * K_PAGE_SIZE, region->flags & MM_REGION_FIXED)) {
return false;
}
region->data.normal.commitPageCount += delta;
space->commit += delta;
if (region->data.normal.commitPageCount > region->pageCount) {
KernelPanic("MMCommitRange - Invalid delta calculation increases region %x commit past page count.\n", region);
}
}
if (!region->data.normal.commit.Set(pageOffset, pageOffset + pageCount, nullptr, true)) {
MMDecommit(delta * K_PAGE_SIZE, region->flags & MM_REGION_FIXED);
region->data.normal.commitPageCount -= delta;
space->commit -= delta;
return false;
}
if (region->flags & MM_REGION_FIXED) {
for (uintptr_t i = pageOffset; i < pageOffset + pageCount; i++) {
// TODO Don't call into MMHandlePageFault. I don't like MM_HANDLE_PAGE_FAULT_LOCK_ACQUIRED.
if (!MMHandlePageFault(space, region->baseAddress + i * K_PAGE_SIZE, MM_HANDLE_PAGE_FAULT_LOCK_ACQUIRED)) {
KernelPanic("MMCommitRange - Unable to fix pages.\n");
}
}
}
return true;
}
bool MMDecommitRange(MMSpace *space, MMRegion *region, uintptr_t pageOffset, size_t pageCount) {
KMutexAssertLocked(&space->reserveMutex);
if (region->flags & MM_REGION_NO_COMMIT_TRACKING) {
KernelPanic("MMDecommitRange - Region does not support commit tracking.\n");
}
if (pageOffset >= region->pageCount || pageCount > region->pageCount - pageOffset) {
KernelPanic("MMDecommitRange - Invalid region offset and page count.\n");
}
if (~region->flags & MM_REGION_NORMAL) {
KernelPanic("MMDecommitRange - Cannot decommit from non-normal region.\n");
}
intptr_t delta = 0;
if (!region->data.normal.commit.Clear(pageOffset, pageOffset + pageCount, &delta, true)) {
return false;
}
if (delta > 0) {
KernelPanic("MMDecommitRange - Invalid delta calculation removing %x, %x from %x.\n", pageOffset, pageCount, region);
}
delta = -delta;
if (region->data.normal.commitPageCount < (size_t) delta) {
KernelPanic("MMDecommitRange - Invalid delta calculation decreases region %x commit below zero.\n", region);
}
// EsPrint("\tdecommit = %x\n", pagesRemoved);
MMDecommit(delta * K_PAGE_SIZE, region->flags & MM_REGION_FIXED);
space->commit -= delta;
region->data.normal.commitPageCount -= delta;
MMArchUnmapPages(space, region->baseAddress + pageOffset * K_PAGE_SIZE, pageCount, MM_UNMAP_PAGES_FREE);
return true;
}
void MMAllowWriteCombiningCaching(MMSpace *space, void *virtualAddress) {
if (!space) space = kernelMMSpace;
KMutexAcquire(&space->reserveMutex);
MMRegion *region = MMFindRegion(space, (uintptr_t) virtualAddress);
if (!(region->flags & MM_REGION_NOT_CACHEABLE)) {
KernelPanic("MMAllowWriteCombiningCaching - Region was cachable.\n");
}
region->flags &= ~MM_REGION_NOT_CACHEABLE;
region->flags |= MM_REGION_WRITE_COMBINING;
MMArchUnmapPages(space, region->baseAddress, region->pageCount, ES_FLAGS_DEFAULT);
KMutexRelease(&space->reserveMutex);
for (uintptr_t i = 0; i < region->pageCount; i++) {
MMHandlePageFault(space, region->baseAddress + i * K_PAGE_SIZE, ES_FLAGS_DEFAULT);
}
}
size_t MMGetRegionPageCount(MMSpace *space, void *address) {
if (!space) space = kernelMMSpace;
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
MMRegion *region = MMFindRegion(space, (uintptr_t) address);
return region->pageCount;
}
void *MMMapPhysical(MMSpace *space, uintptr_t offset, size_t bytes, uint64_t caching) {
if (!space) space = kernelMMSpace;
uintptr_t offset2 = offset & (K_PAGE_SIZE - 1);
offset -= offset2;
if (offset2) bytes += K_PAGE_SIZE;
MMRegion *region;
{
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
region = MMReserve(space, bytes, MM_REGION_PHYSICAL | MM_REGION_FIXED | caching);
if (!region) return nullptr;
region->data.physical.offset = offset;
}
for (uintptr_t i = 0; i < region->pageCount; i++) {
MMHandlePageFault(space, region->baseAddress + i * K_PAGE_SIZE, ES_FLAGS_DEFAULT);
}
return (uint8_t *) region->baseAddress + offset2;
}
bool MMSharedResizeRegion(MMSharedRegion *region, size_t sizeBytes) {
KMutexAcquire(&region->mutex);
EsDefer(KMutexRelease(&region->mutex));
sizeBytes = (sizeBytes + K_PAGE_SIZE - 1) & ~(K_PAGE_SIZE - 1);
size_t pages = sizeBytes / K_PAGE_SIZE;
size_t oldPages = region->sizeBytes / K_PAGE_SIZE;
void *oldData = region->data;
void *newData = EsHeapAllocate(pages * sizeof(void *), true, K_CORE);
if (!newData && pages) {
return false;
}
if (oldPages > pages) {
MMDecommit(K_PAGE_SIZE * (oldPages - pages), true);
} else if (pages > oldPages) {
if (!MMCommit(K_PAGE_SIZE * (pages - oldPages), true)) {
EsHeapFree(newData, pages * sizeof(void *), K_CORE);
return false;
}
}
region->sizeBytes = sizeBytes;
region->data = newData;
// The old shared memory region was empty.
if (!oldData) return true;
if (oldPages > pages) {
for (uintptr_t i = pages; i < oldPages; i++) {
uintptr_t *addresses = (uintptr_t *) oldData;
uintptr_t address = addresses[i];
if (address & MM_SHARED_ENTRY_PRESENT) MMPhysicalFree(address);
}
}
uintptr_t copy = oldPages > pages ? pages : oldPages;
EsMemoryCopy(region->data, oldData, sizeof(void *) * copy);
EsHeapFree(oldData, oldPages * sizeof(void *), K_CORE);
return true;
}
MMSharedRegion *MMSharedCreateRegion(size_t sizeBytes, bool fixed, uintptr_t below) {
if (!sizeBytes) return nullptr;
MMSharedRegion *region = (MMSharedRegion *) EsHeapAllocate(sizeof(MMSharedRegion), true, K_CORE);
if (!region) return nullptr;
region->handles = 1;
if (!MMSharedResizeRegion(region, sizeBytes)) {
EsHeapFree(region, 0, K_CORE);
return nullptr;
}
if (fixed) {
for (uintptr_t i = 0; i < region->sizeBytes >> K_PAGE_BITS; i++) {
((uintptr_t *) region->data)[i] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_ZEROED, 1, 1, below) | MM_SHARED_ENTRY_PRESENT;
}
}
return region;
}
void MMSharedDestroyRegion(MMSharedRegion *region) {
MMSharedResizeRegion(region, 0); // TODO Check leaks.
EsHeapFree(region, 0, K_CORE);
}
void *MMStandardAllocate(MMSpace *space, size_t bytes, uint32_t flags, void *baseAddress, bool commitAll) {
if (!space) space = kernelMMSpace;
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
MMRegion *region = MMReserve(space, bytes, flags | MM_REGION_NORMAL, (uintptr_t) baseAddress, true);
if (!region) return nullptr;
if (commitAll) {
if (!MMCommitRange(space, region, 0, region->pageCount)) {
MMUnreserve(space, region, false /* No pages have been mapped. */);
return nullptr;
}
}
return (void *) region->baseAddress;
}
bool MMFree(MMSpace *space, void *address, size_t expectedSize, bool userOnly) {
if (!space) space = kernelMMSpace;
MMSharedRegion *sharedRegionToFree = nullptr;
FSFile *nodeToFree = nullptr;
uint64_t fileHandleFlags = 0;
{
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
MMRegion *region = MMFindRegion(space, (uintptr_t) address);
if (!region) {
return false;
}
if (userOnly && (~region->flags & MM_REGION_USER)) {
KernelLog(LOG_ERROR, "Memory", "attempt to free non-user region",
"MMFree - A user process is attempting to free a region %x that is not marked with MM_REGION_USER.\n", region);
return false;
}
if (!KWriterLockTake(&region->data.pin, K_LOCK_EXCLUSIVE, true /* poll */)) {
KernelLog(LOG_ERROR, "Memory", "attempt to free in use region",
"MMFree - Attempting to free a region %x that is currently in by a system call.\n", region);
return false;
}
if (region->baseAddress != (uintptr_t) address && (~region->flags & MM_REGION_PHYSICAL /* physical region bases are not page aligned */)) {
KernelLog(LOG_ERROR, "Memory", "incorrect base address", "MMFree - Passed the address %x to free region %x, which has baseAddress of %x.\n",
address, region, region->baseAddress);
return false;
}
if (expectedSize && (expectedSize + K_PAGE_SIZE - 1) / K_PAGE_SIZE != region->pageCount) {
KernelLog(LOG_ERROR, "Memory", "incorrect free size", "MMFree - The region page count is %d, but the expected free size is %d.\n",
region->pageCount, (expectedSize + K_PAGE_SIZE - 1) / K_PAGE_SIZE);
return false;
}
bool unmapPages = true;
if (region->flags & MM_REGION_NORMAL) {
if (!MMDecommitRange(space, region, 0, region->pageCount)) {
KernelPanic("MMFree - Could not decommit entire region %x (should not fail).\n", region);
}
if (region->data.normal.commitPageCount) {
KernelPanic("MMFree - After decommiting range covering the entire region (%x), some pages were still commited.\n", region);
}
region->data.normal.commit.ranges.Free();
unmapPages = false;
} else if (region->flags & MM_REGION_SHARED) {
sharedRegionToFree = region->data.shared.region;
} else if (region->flags & MM_REGION_FILE) {
MMArchUnmapPages(space, region->baseAddress, region->pageCount, MM_UNMAP_PAGES_FREE_COPIED | MM_UNMAP_PAGES_BALANCE_FILE);
unmapPages = false;
FSFile *node = region->data.file.node;
EsFileOffset removeStart = RoundDown(region->data.file.offset, (EsFileOffset) K_PAGE_SIZE);
EsFileOffset removeEnd = RoundUp(removeStart + (region->pageCount << K_PAGE_BITS) - region->data.file.zeroedBytes, (EsFileOffset) K_PAGE_SIZE);
nodeToFree = node;
fileHandleFlags = region->data.file.fileHandleFlags;
KMutexAcquire(&node->cache.cachedSectionsMutex);
CCSpaceUncover(&node->cache, removeStart, removeEnd);
KMutexRelease(&node->cache.cachedSectionsMutex);
if (region->flags & MM_REGION_COPY_ON_WRITE) {
MMDecommit(region->pageCount << K_PAGE_BITS, false);
}
} else if (region->flags & MM_REGION_PHYSICAL) {
} else if (region->flags & MM_REGION_GUARD) {
KernelLog(LOG_ERROR, "Memory", "attempt to free guard region", "MMFree - Attempting to free a guard region!\n");
return false;
} else {
KernelPanic("MMFree - Unsupported region type.\n");
}
MMUnreserve(space, region, unmapPages);
}
if (sharedRegionToFree) CloseHandleToObject(sharedRegionToFree, KERNEL_OBJECT_SHMEM);
if (nodeToFree && fileHandleFlags) CloseHandleToObject(nodeToFree, KERNEL_OBJECT_NODE, fileHandleFlags);
return true;
}
bool MMSpaceInitialise(MMSpace *space) {
// EsPrint("... Committed %d/%d fixed and %d pageable out of a maximum %d.\n", pmm.commitFixed, pmm.commitFixedLimit, pmm.commitPageable, pmm.commitLimit);
space->user = true;
MMRegion *region = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
if (!region) {
return false;
}
if (!MMArchInitialiseUserSpace(space, region)) {
EsHeapFree(region, sizeof(MMRegion), K_CORE);
return false;
}
TreeInsert(&space->freeRegionsBase, &region->itemBase, region, MakeShortKey(region->baseAddress));
TreeInsert(&space->freeRegionsSize, &region->itemSize, region, MakeShortKey(region->pageCount), AVL_DUPLICATE_KEYS_ALLOW);
return true;
}
void MMSpaceDestroy(MMSpace *space) {
LinkedItem<MMRegion> *item = space->usedRegionsNonGuard.firstItem;
while (item) {
MMRegion *region = item->thisItem;
item = item->nextItem;
MMFree(space, (void *) region->baseAddress);
}
while (true) {
AVLItem<MMRegion> *item = TreeFind(&space->freeRegionsBase, MakeShortKey(0), TREE_SEARCH_SMALLEST_ABOVE_OR_EQUAL);
if (!item) break;
TreeRemove(&space->freeRegionsBase, &item->thisItem->itemBase);
TreeRemove(&space->freeRegionsSize, &item->thisItem->itemSize);
EsHeapFree(item->thisItem, sizeof(MMRegion), K_CORE);
}
MMArchFreeVAS(space);
}
bool MMUnmapFilePage(uintptr_t frameNumber) {
KMutexAssertLocked(&pmm.pageFrameMutex);
MMPageFrame *frame = pmm.pageFrames + frameNumber;
if (frame->state != MMPageFrame::ACTIVE || !frame->active.references) {
KernelPanic("MMUnmapFilePage - Corrupt page frame database (%d/%x).\n", frameNumber, frame);
}
// Decrease the reference count.
frame->active.references--;
if (frame->active.references) {
return false;
}
// If there are no more references, then the frame can be moved to the standby or modified list.
// EsPrint("Unmap file page: %x\n", frameNumber << K_PAGE_BITS);
frame->state = MMPageFrame::STANDBY;
pmm.countStandbyPages++;
if (*frame->cacheReference != ((frameNumber << K_PAGE_BITS) | MM_SHARED_ENTRY_PRESENT)) {
KernelPanic("MMUnmapFilePage - Corrupt shared reference back pointer in frame %x.\n", frame);
}
frame->list.next = pmm.firstStandbyPage;
frame->list.previous = &pmm.firstStandbyPage;
if (pmm.firstStandbyPage) pmm.pageFrames[pmm.firstStandbyPage].list.previous = &frame->list.next;
if (!pmm.lastStandbyPage) pmm.lastStandbyPage = frameNumber;
pmm.firstStandbyPage = frameNumber;
MMUpdateAvailablePageCount(true);
pmm.countActivePages--;
return true;
}
void MMBalanceThread() {
size_t targetAvailablePages = 0;
while (true) {
#if 1
if (MM_AVAILABLE_PAGES() >= targetAvailablePages) {
// Wait for there to be a low number of available pages.
KEventWait(&pmm.availableLow);
targetAvailablePages = MM_LOW_AVAILABLE_PAGES_THRESHOLD + MM_PAGES_TO_FIND_BALANCE;
}
#else
// Test the balance thread works correctly by running it constantly.
targetAvailablePages = MM_AVAILABLE_PAGES() * 2;
#endif
// EsPrint("--> Balance!\n");
// Find a process to balance.
KMutexAcquire(&scheduler.allProcessesMutex);
Process *process = pmm.nextProcessToBalance ?: scheduler.allProcesses.firstItem->thisItem;
pmm.nextProcessToBalance = process->allItem.nextItem ? process->allItem.nextItem->thisItem : nullptr;
OpenHandleToObject(process, KERNEL_OBJECT_PROCESS, ES_FLAGS_DEFAULT);
KMutexRelease(&scheduler.allProcessesMutex);
// For every memory region...
MMSpace *space = process->vmm;
ThreadSetTemporaryAddressSpace(space);
KMutexAcquire(&space->reserveMutex);
LinkedItem<MMRegion> *item = pmm.nextRegionToBalance ? &pmm.nextRegionToBalance->itemNonGuard : space->usedRegionsNonGuard.firstItem;
while (item && MM_AVAILABLE_PAGES() < targetAvailablePages) {
MMRegion *region = item->thisItem;
// EsPrint("process = %x, region = %x, offset = %x\n", process, region, pmm.balanceResumePosition);
KMutexAcquire(&region->data.mapMutex);
bool canResume = false;
if (region->flags & MM_REGION_FILE) {
canResume = true;
MMArchUnmapPages(space,
region->baseAddress, region->pageCount,
MM_UNMAP_PAGES_BALANCE_FILE,
targetAvailablePages - MM_AVAILABLE_PAGES(), &pmm.balanceResumePosition);
} else if (region->flags & MM_REGION_CACHE) {
// TODO Trim the cache's active sections and cached sections lists.
KMutexAcquire(&activeSectionManager.mutex);
LinkedItem<CCActiveSection> *item = activeSectionManager.lruList.firstItem;
while (item && MM_AVAILABLE_PAGES() < targetAvailablePages) {
CCActiveSection *section = item->thisItem;
if (section->cache && section->referencedPageCount) CCDereferenceActiveSection(section);
item = item->nextItem;
}
KMutexRelease(&activeSectionManager.mutex);
}
KMutexRelease(&region->data.mapMutex);
if (MM_AVAILABLE_PAGES() >= targetAvailablePages && canResume) {
// We have enough to pause.
break;
}
item = item->nextItem;
pmm.balanceResumePosition = 0;
}
if (item) {
// Continue with this region next time.
pmm.nextRegionToBalance = item->thisItem;
pmm.nextProcessToBalance = process;
// EsPrint("will resume at %x\n", pmm.balanceResumePosition);
} else {
// Go to the next process.
pmm.nextRegionToBalance = nullptr;
pmm.balanceResumePosition = 0;
}
KMutexRelease(&space->reserveMutex);
ThreadSetTemporaryAddressSpace(nullptr);
CloseHandleToObject(process, KERNEL_OBJECT_PROCESS);
}
}
void MMZeroPageThread() {
while (true) {
KEventWait(&pmm.zeroPageEvent);
KEventWait(&pmm.availableNotCritical);
bool done = false;
while (!done) {
uintptr_t pages[PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES];
int i = 0;
{
KMutexAcquire(&pmm.pageFrameMutex);
EsDefer(KMutexRelease(&pmm.pageFrameMutex));
for (; i < PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES; i++) {
if (pmm.firstFreePage) {
pages[i] = pmm.firstFreePage;
MMPhysicalActivatePages(pages[i], 1, ES_FLAGS_DEFAULT);
} else {
done = true;
break;
}
MMPageFrame *frame = pmm.pageFrames + pages[i];
frame->state = MMPageFrame::ACTIVE;
pmm.freeOrZeroedPageBitset.Take(pages[i]);
}
}
for (int j = 0; j < i; j++) pages[j] <<= K_PAGE_BITS;
if (i) PMZero(pages, i, false);
KMutexAcquire(&pmm.pageFrameMutex);
pmm.countActivePages -= i;
while (i--) {
MMPhysicalInsertZeroedPage(pages[i] >> K_PAGE_BITS);
}
KMutexRelease(&pmm.pageFrameMutex);
}
}
}
#if 0
void PMCheckZeroed(uintptr_t page) {
pmm.pmManipulationLock.Acquire();
{
MMSpace *vas = coreMMSpace;
void *region = pmm.pmManipulationRegion;
vas->data.mapMutex.Acquire();
MMArchMapPage(vas, page, (uintptr_t) region, MM_MAP_PAGE_OVERWRITE);
vas->data.mapMutex.Release();
pmm.pmManipulationProcessorLock.Acquire();
ProcessorInvalidatePage((uintptr_t) region);
for (uintptr_t i = 0; i < K_PAGE_SIZE; i++) {
if (((uint8_t *) region)[i]) {
KernelPanic("PMCheckZeroed - Supposedly zeroed page %x was not zeroed.\n", page);
}
}
pmm.pmManipulationProcessorLock.Release();
}
pmm.pmManipulationLock.Release();
}
#endif
void PMZeroPartial(uintptr_t page, uintptr_t start, uintptr_t end) {
KMutexAcquire(&pmm.pmManipulationLock);
MMSpace *vas = coreMMSpace;
void *region = pmm.pmManipulationRegion;
MMArchMapPage(vas, page, (uintptr_t) region, MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES);
KSpinlockAcquire(&pmm.pmManipulationProcessorLock);
ProcessorInvalidatePage((uintptr_t) region);
EsMemoryZero((uint8_t *) region + start, end - start);
KSpinlockRelease(&pmm.pmManipulationProcessorLock);
KMutexRelease(&pmm.pmManipulationLock);
}
void PMZero(uintptr_t *pages, size_t pageCount, bool contiguous) {
KMutexAcquire(&pmm.pmManipulationLock);
repeat:;
size_t doCount = pageCount > PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES ? PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES : pageCount;
pageCount -= doCount;
{
MMSpace *vas = coreMMSpace;
void *region = pmm.pmManipulationRegion;
for (uintptr_t i = 0; i < doCount; i++) {
MMArchMapPage(vas, contiguous ? pages[0] + (i << K_PAGE_BITS) : pages[i],
(uintptr_t) region + K_PAGE_SIZE * i, MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES);
}
KSpinlockAcquire(&pmm.pmManipulationProcessorLock);
for (uintptr_t i = 0; i < doCount; i++) {
ProcessorInvalidatePage((uintptr_t) region + i * K_PAGE_SIZE);
}
EsMemoryZero(region, doCount * K_PAGE_SIZE);
KSpinlockRelease(&pmm.pmManipulationProcessorLock);
}
if (pageCount) {
if (!contiguous) pages += PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES;
goto repeat;
}
// if (pageNumbers) EsPrint("w%d\n", pmm.pmManipulationLock.blockedThreads.count);
KMutexRelease(&pmm.pmManipulationLock);
}
void PMCopy(uintptr_t page, void *_source, size_t pageCount) {
uint8_t *source = (uint8_t *) _source;
KMutexAcquire(&pmm.pmManipulationLock);
repeat:;
size_t doCount = pageCount > PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES ? PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES : pageCount;
pageCount -= doCount;
{
MMSpace *vas = coreMMSpace;
void *region = pmm.pmManipulationRegion;
for (uintptr_t i = 0; i < doCount; i++) {
MMArchMapPage(vas, page + K_PAGE_SIZE * i, (uintptr_t) region + K_PAGE_SIZE * i,
MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES);
}
KSpinlockAcquire(&pmm.pmManipulationProcessorLock);
for (uintptr_t i = 0; i < doCount; i++) {
ProcessorInvalidatePage((uintptr_t) region + i * K_PAGE_SIZE);
}
EsMemoryCopy(region, source, doCount * K_PAGE_SIZE);
KSpinlockRelease(&pmm.pmManipulationProcessorLock);
}
if (pageCount) {
page += PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE;
source += PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE;
goto repeat;
}
KMutexRelease(&pmm.pmManipulationLock);
}
void PMRead(uintptr_t page, void *_source, size_t pageCount) {
uint8_t *source = (uint8_t *) _source;
KMutexAcquire(&pmm.pmManipulationLock);
repeat:;
size_t doCount = pageCount > PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES ? PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES : pageCount;
pageCount -= doCount;
{
MMSpace *vas = coreMMSpace;
void *region = pmm.pmManipulationRegion;
for (uintptr_t i = 0; i < doCount; i++) {
MMArchMapPage(vas, page + K_PAGE_SIZE * i, (uintptr_t) region + K_PAGE_SIZE * i,
MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES);
}
KSpinlockAcquire(&pmm.pmManipulationProcessorLock);
for (uintptr_t i = 0; i < doCount; i++) {
ProcessorInvalidatePage((uintptr_t) region + i * K_PAGE_SIZE);
}
EsMemoryCopy(source, region, doCount * K_PAGE_SIZE);
KSpinlockRelease(&pmm.pmManipulationProcessorLock);
}
if (pageCount) {
page += PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE;
source += PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE;
goto repeat;
}
KMutexRelease(&pmm.pmManipulationLock);
}
void *Pool::Add(size_t _elementSize) {
KMutexAcquire(&mutex);
EsDefer(KMutexRelease(&mutex));
if (elementSize && _elementSize != elementSize) KernelPanic("Pool::Add - Pool element size mismatch.\n");
elementSize = _elementSize;
void *address;
if (cacheEntries) {
address = cache[--cacheEntries];
EsMemoryZero(address, elementSize);
} else {
address = EsHeapAllocate(elementSize, true, K_FIXED);
}
return address;
}
void Pool::Remove(void *address) {
KMutexAcquire(&mutex);
EsDefer(KMutexRelease(&mutex));
if (!address) return;
#if 1
if (cacheEntries == POOL_CACHE_COUNT) {
EsHeapFree(address, elementSize, K_FIXED);
} else {
cache[cacheEntries++] = address;
}
#else
EsHeapFree(address, elementSize);
#endif
}
MMRegion *MMFindAndPinRegion(MMSpace *space, uintptr_t address, uintptr_t size) {
if (address + size < address) {
return nullptr;
}
KMutexAcquire(&space->reserveMutex);
EsDefer(KMutexRelease(&space->reserveMutex));
MMRegion *region = MMFindRegion(space, address);
if (!region) {
return nullptr;
}
if (region->baseAddress > address) {
return nullptr;
}
if (region->baseAddress + region->pageCount * K_PAGE_SIZE < address + size) {
return nullptr;
}
if (!KWriterLockTake(&region->data.pin, K_LOCK_SHARED, true /* poll */)) {
return nullptr;
}
return region;
}
void MMUnpinRegion(MMSpace *space, MMRegion *region) {
KMutexAcquire(&space->reserveMutex);
KWriterLockReturn(&region->data.pin, K_LOCK_SHARED);
KMutexRelease(&space->reserveMutex);
}
bool MMPhysicalAllocateAndMap(size_t sizeBytes, size_t alignmentBytes, size_t maximumBits, bool zeroed,
uint64_t caching, uint8_t **_virtualAddress, uintptr_t *_physicalAddress) {
if (!sizeBytes) sizeBytes = 1;
if (!alignmentBytes) alignmentBytes = 1;
bool noBelow = false;
#ifdef ES_BITS_32
if (!maximumBits || maximumBits >= 32) noBelow = true;
#endif
#ifdef ES_BITS_64
if (!maximumBits || maximumBits >= 64) noBelow = true;
#endif
uintptr_t sizePages = (sizeBytes + K_PAGE_SIZE - 1) >> K_PAGE_BITS;
uintptr_t physicalAddress = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_CAN_FAIL | MM_PHYSICAL_ALLOCATE_COMMIT_NOW,
sizePages, (alignmentBytes + K_PAGE_SIZE - 1) >> K_PAGE_BITS,
noBelow ? 0 : ((size_t) 1 << maximumBits));
if (!physicalAddress) {
return false;
}
void *virtualAddress = MMMapPhysical(kernelMMSpace, physicalAddress, sizeBytes, caching);
if (!virtualAddress) {
MMPhysicalFree(physicalAddress, false, sizePages);
return false;
}
if (zeroed) {
EsMemoryZero(virtualAddress, sizeBytes);
}
*_virtualAddress = (uint8_t *) virtualAddress;
*_physicalAddress = physicalAddress;
return true;
}
void MMPhysicalFreeAndUnmap(void *virtualAddress, uintptr_t physicalAddress) {
KMutexAcquire(&kernelMMSpace->reserveMutex);
MMRegion *region = MMFindRegion(kernelMMSpace, (uintptr_t) virtualAddress);
if (!region || (~region->flags & MM_REGION_PHYSICAL) || region->data.physical.offset != physicalAddress) {
KernelPanic("MMPhysicalFreeAndUnmap - Virtual address %x did not point to a region of physical memory mapping %x.\n", virtualAddress, physicalAddress);
}
size_t pageCount = region->pageCount;
MMUnreserve(kernelMMSpace, region, true);
KMutexRelease(&kernelMMSpace->reserveMutex);
MMPhysicalFree(physicalAddress, false, pageCount);
}
MMSpace *MMGetKernelSpace() {
return kernelMMSpace;
}
MMSpace *MMGetCurrentProcessSpace() {
return GetCurrentThread()->process->vmm;
}
bool MMFaultRange(uintptr_t address, uintptr_t byteCount, uint32_t flags = ES_FLAGS_DEFAULT) {
uintptr_t start = address & ~(K_PAGE_SIZE - 1);
uintptr_t end = (address + byteCount - 1) & ~(K_PAGE_SIZE - 1);
for (uintptr_t page = start; page <= end; page += K_PAGE_SIZE) {
if (!MMArchHandlePageFault(page, flags)) {
return false;
}
}
return true;
}
void MMRemapPhysical(MMSpace *space, const void *virtualAddress, uintptr_t newPhysicalAddress) {
if (ProcessorAreInterruptsEnabled()) {
KernelPanic("MMRemapPhysical - Cannot remap address with interrupts enabled (does not invalidate the page on other processors).\n");
}
MMArchMapPage(space, newPhysicalAddress, (uintptr_t) virtualAddress, MM_MAP_PAGE_OVERWRITE | MM_MAP_PAGE_NO_NEW_TABLES);
}
void MMObjectCacheInsert(MMObjectCache *cache, MMObjectCacheItem *item) {
KSpinlockAcquire(&cache->lock);
cache->items.Insert(item, false /* end */);
cache->count++;
__sync_fetch_and_add(&pmm.approximateTotalObjectCacheBytes, cache->averageObjectBytes);
if (MM_OBJECT_CACHE_SHOULD_TRIM()) {
KEventSet(&pmm.trimObjectCaches, true);
}
KSpinlockRelease(&cache->lock);
}
void MMObjectCacheRemove(MMObjectCache *cache, MMObjectCacheItem *item, bool alreadyLocked) {
if (!alreadyLocked) KSpinlockAcquire(&cache->lock);
else KSpinlockAssertLocked(&cache->lock);
item->Remove();
cache->count--;
__sync_fetch_and_sub(&pmm.approximateTotalObjectCacheBytes, cache->averageObjectBytes);
if (!alreadyLocked) KSpinlockRelease(&cache->lock);
}
MMObjectCacheItem *MMObjectCacheRemoveLRU(MMObjectCache *cache) {
MMObjectCacheItem *item = nullptr;
KSpinlockAcquire(&cache->lock);
if (cache->count) {
item = cache->items.first;
MMObjectCacheRemove(cache, item, true);
}
KSpinlockRelease(&cache->lock);
return item;
}
void MMObjectCacheRegister(MMObjectCache *cache, bool (*trim)(MMObjectCache *), size_t averageObjectBytes) {
KMutexAcquire(&pmm.objectCacheListMutex);
cache->trim = trim;
cache->averageObjectBytes = averageObjectBytes;
cache->item.thisItem = cache;
pmm.objectCacheList.InsertEnd(&cache->item);
KMutexRelease(&pmm.objectCacheListMutex);
}
void MMObjectCacheUnregister(MMObjectCache *cache) {
KMutexAcquire(&pmm.objectCacheListMutex);
pmm.objectCacheList.Remove(&cache->item);
KMutexRelease(&pmm.objectCacheListMutex);
// Wait for any trim threads still using the cache to finish.
KWriterLockTake(&cache->trimLock, K_LOCK_EXCLUSIVE);
KWriterLockReturn(&cache->trimLock, K_LOCK_EXCLUSIVE);
}
void MMObjectCacheFlush(MMObjectCache *cache) {
if (cache->item.list) KernelPanic("MMObjectCacheFlush - Cache %x must be unregistered before flushing.\n", cache);
// Wait for any trim threads still using the cache to finish.
KWriterLockTake(&cache->trimLock, K_LOCK_EXCLUSIVE);
// Trim the cache until it is empty.
// The trim callback is allowed to increase cache->count,
// but nobody else should be increasing it once it has been unregistered.
while (cache->count) cache->trim(cache);
// Return the trim lock.
KWriterLockReturn(&cache->trimLock, K_LOCK_EXCLUSIVE);
}
void MMObjectCacheTrimThread() {
MMObjectCache *cache = nullptr;
while (true) {
while (!MM_OBJECT_CACHE_SHOULD_TRIM()) {
KEventReset(&pmm.trimObjectCaches);
KEventWait(&pmm.trimObjectCaches);
}
KMutexAcquire(&pmm.objectCacheListMutex);
// TODO Is there a faster way to find the next cache?
// This needs to work with multiple producers and consumers.
// And I don't want to put the caches in an array, because then registering a cache could fail.
MMObjectCache *previousCache = cache;
cache = nullptr;
LinkedItem<MMObjectCache> *item = pmm.objectCacheList.firstItem;
while (item) {
if (item->thisItem == previousCache && item->nextItem) {
cache = item->nextItem->thisItem;
break;
}
item = item->nextItem;
}
if (!cache && pmm.objectCacheList.firstItem) {
cache = pmm.objectCacheList.firstItem->thisItem;
}
if (cache) {
KWriterLockTake(&cache->trimLock, K_LOCK_SHARED);
}
KMutexRelease(&pmm.objectCacheListMutex);
if (!cache) {
continue;
}
for (uintptr_t i = 0; i < MM_OBJECT_CACHE_TRIM_GROUP_COUNT; i++) {
if (!cache->trim(cache)) {
break;
}
}
KWriterLockReturn(&cache->trimLock, K_LOCK_SHARED);
}
}
void MMSpaceOpenReference(MMSpace *space) {
if (space == kernelMMSpace) {
return;
}
if (space->referenceCount < 1) {
KernelPanic("MMSpaceOpenReference - Space %x has invalid reference count.\n", space);
}
__sync_fetch_and_add(&space->referenceCount, 1);
}
void MMSpaceCloseReference(MMSpace *space) {
if (space == kernelMMSpace) {
return;
}
if (space->referenceCount < 1) {
KernelPanic("MMSpaceCloseReference - Space %x has invalid reference count.\n", space);
}
if (__sync_fetch_and_sub(&space->referenceCount, 1) > 1) {
return;
}
KRegisterAsyncTask(&space->removeAsyncTask, [] (KAsyncTask *task) {
MMSpace *space = EsContainerOf(MMSpace, removeAsyncTask, task);
MMArchFinalizeVAS(space);
scheduler.mmSpacePool.Remove(space);
});
}
void MMInitialise() {
{
// Initialise coreMMSpace and kernelMMSpace.
mmCoreRegions[0].core.used = false;
mmCoreRegionCount = 1;
MMArchInitialise();
MMRegion *region = (MMRegion *) EsHeapAllocate(sizeof(MMRegion), true, K_CORE);
region->baseAddress = MM_KERNEL_SPACE_START;
region->pageCount = MM_KERNEL_SPACE_SIZE / K_PAGE_SIZE;
TreeInsert(&kernelMMSpace->freeRegionsBase, &region->itemBase, region, MakeShortKey(region->baseAddress));
TreeInsert(&kernelMMSpace->freeRegionsSize, &region->itemSize, region, MakeShortKey(region->pageCount), AVL_DUPLICATE_KEYS_ALLOW);
}
{
// Initialise physical memory management.
KMutexAcquire(&kernelMMSpace->reserveMutex);
pmm.pmManipulationRegion = (void *) MMReserve(kernelMMSpace, PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE, ES_FLAGS_DEFAULT)->baseAddress;
KMutexRelease(&kernelMMSpace->reserveMutex);
// 1 extra for the top page, then round up so the page bitset is byte-aligned.
uintptr_t pageFrameDatabaseCount = (MMArchGetPhysicalMemoryHighest() + (K_PAGE_SIZE << 3)) >> K_PAGE_BITS;
pmm.pageFrames = (MMPageFrame *) MMStandardAllocate(kernelMMSpace, pageFrameDatabaseCount * sizeof(MMPageFrame), MM_REGION_FIXED);
pmm.freeOrZeroedPageBitset.Initialise(pageFrameDatabaseCount, true);
pmm.pageFrameDatabaseCount = pageFrameDatabaseCount; // Only set this after the database is ready, or it may be accessed mid-allocation!
MMPhysicalInsertFreePagesStart();
uint64_t commitLimit = MMArchPopulatePageFrameDatabase();
MMPhysicalInsertFreePagesEnd();
pmm.pageFrameDatabaseInitialised = true;
pmm.commitLimit = pmm.commitFixedLimit = commitLimit;
KernelLog(LOG_INFO, "Memory", "pmm initialised", "MMInitialise - PMM initialised with a fixed commit limit of %d pages.\n", pmm.commitLimit);
}
{
// Initialise file cache.
CCInitialise();
}
{
// Create threads.
pmm.zeroPageEvent.autoReset = true;
MMCommit(PHYSICAL_MEMORY_MANIPULATION_REGION_PAGES * K_PAGE_SIZE, true);
pmm.zeroPageThread = ThreadSpawn("MMZero", (uintptr_t) MMZeroPageThread, 0, SPAWN_THREAD_LOW_PRIORITY);
ThreadSpawn("MMBalance", (uintptr_t) MMBalanceThread, 0, ES_FLAGS_DEFAULT)->isPageGenerator = true;
ThreadSpawn("MMObjTrim", (uintptr_t) MMObjectCacheTrimThread, 0, ES_FLAGS_DEFAULT);
}
{
// Create the global data shared region, and the API table region.
mmAPITableRegion = MMSharedCreateRegion(0xF000, false, 0);
mmGlobalDataRegion = MMSharedCreateRegion(sizeof(GlobalData), false, 0);
mmGlobalData = (GlobalData *) MMMapShared(kernelMMSpace, mmGlobalDataRegion, 0, sizeof(GlobalData), MM_REGION_FIXED);
MMFaultRange((uintptr_t) mmGlobalData, sizeof(GlobalData), MM_HANDLE_PAGE_FAULT_FOR_SUPERVISOR);
}
}
#endif
View Git Blame Copy Permalink