From ed038d10591522a49f72cbb2c3033d9e1980f615 Mon Sep 17 00:00:00 2001 From: nakst <> Date: Mon, 1 Nov 2021 19:32:56 +0000 Subject: [PATCH] x86_32 implementation for kernel --- arch/x86_32/api.s | 3 +- arch/x86_32/kernel.cpp | 522 +++++++++++++--------- arch/x86_32/kernel.s | 549 ++++++++++++++++++++++- arch/x86_64.h | 83 ---- arch/x86_64/kernel.cpp | 932 +++----------------------------------- arch/x86_64/kernel.s | 256 +++-------- arch/x86_pc.cpp | 990 +++++++++++++++++++++++++++++++++++++++++ arch/x86_pc.h | 43 ++ drivers/acpi.cpp | 2 + drivers/ahci.cpp | 19 +- drivers/esfs2.cpp | 18 +- drivers/nvme.cpp | 2 +- drivers/pci.cpp | 4 + drivers/xhci.cpp | 2 +- kernel/kernel.h | 1 - kernel/memory.cpp | 8 +- kernel/scheduler.cpp | 5 + kernel/terminal.cpp | 1 - shared/esfs2.h | 3 + start.sh | 4 +- util/build.c | 19 +- util/build_core.c | 1 + 22 files changed, 2066 insertions(+), 1401 deletions(-) delete mode 100644 arch/x86_64.h create mode 100644 arch/x86_pc.cpp diff --git a/arch/x86_32/api.s b/arch/x86_32/api.s index d3f51af..c1ee8d0 100644 --- a/arch/x86_32/api.s +++ b/arch/x86_32/api.s @@ -27,12 +27,11 @@ EsCRTsqrtf: [global ProcessorReadTimeStamp] ProcessorReadTimeStamp: - ; TODO + rdtsc ret [global ProcessorCheckStackAlignment] ProcessorCheckStackAlignment: - ; TODO ret [global ProcessorTLSRead] diff --git a/arch/x86_32/kernel.cpp b/arch/x86_32/kernel.cpp index 46bc9c6..fccddab 100755 --- a/arch/x86_32/kernel.cpp +++ b/arch/x86_32/kernel.cpp @@ -1,14 +1,36 @@ +// TODO Support for systems without MMX/SSE. +// TODO Support for systems without an APIC. + #ifndef IMPLEMENTATION struct MMArchVAS { + // NOTE Must be first in the structure. See ProcessorSetAddressSpace and ArchSwitchContext. + uintptr_t cr3; + + // Each process has a 32-bit address space. + // That's 2^20 pages. + // That's 2^10 L1 page tables. 2^7 bytes of bitset. + // Tracking of the committed L1 tables is done in l1Commit. +#define L1_COMMIT_SIZE_BYTES (1 << 7) + uint8_t l1Commit[L1_COMMIT_SIZE_BYTES]; + size_t pageTablesCommitted; + size_t pageTablesActive; + + // TODO Consider core/kernel mutex consistency? I think it's fine, but... + KMutex mutex; // Acquire to modify the page tables. }; -#define MM_KERNEL_SPACE_START (0xC0000000) -#define MM_KERNEL_SPACE_SIZE (0xE0000000 - 0xC0000000) +#define MM_KERNEL_SPACE_START (0xC4000000) +#define MM_KERNEL_SPACE_SIZE (0xE0000000 - 0xC4000000) #define MM_MODULES_START (0xE0000000) -#define MM_MODULES_SIZE (0xE8000000 - 0xE0000000) -#define MM_CORE_REGIONS_START (0xE8000000) -#define MM_CORE_REGIONS_COUNT ((0xEC000000 - 0xE8000000) / sizeof(MMRegion)) +#define MM_MODULES_SIZE (0xE6000000 - 0xE0000000) +#define MM_CORE_REGIONS_START (0xE6000000) +#define MM_CORE_REGIONS_COUNT ((0xE7000000 - 0xE6000000) / sizeof(MMRegion)) +#define MM_CORE_SPACE_START (0xE7000000) +#define MM_CORE_SPACE_SIZE (0xEC000000 - 0xE7000000) +#define LOW_MEMORY_MAP_START (0xEC000000) +#define LOW_MEMORY_LIMIT (0x00400000) // The first 4MB - 4KB is mapped here; the last 4KB map to the local APIC. +#define LOCAL_APIC_BASE (0xEC3FF000) #define ArchCheckBundleHeader() (header.mapAddress >= 0xC0000000ULL || header.mapAddress < 0x1000 || fileSize > 0x10000000ULL) #define ArchCheckELFHeader() (header->virtualAddress >= 0xC0000000ULL || header->virtualAddress < 0x1000 || header->segmentSize > 0x10000000ULL) @@ -20,266 +42,328 @@ struct MMArchVAS { #ifdef IMPLEMENTATION -#define KERNEL_PANIC_IPI (0) // NMIs ignore the interrupt vector. -size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi = false, int processorID = -1); // Returns the number of processors the IPI was *not* sent to. -void ProcessorReset(); -uintptr_t ArchFindRootSystemDescriptorPointer(); -void ArchStartupApplicationProcessors(); -uint32_t bootloaderID; +// Recursive page table mapping in slot 0x1FF. +#define PAGE_TABLE_L2 ((volatile uint32_t *) 0xFFFFF000) +#define PAGE_TABLE_L1 ((volatile uint32_t *) 0xFFC00000) +#define ENTRIES_PER_PAGE_TABLE (1024) +#define ENTRIES_PER_PAGE_TABLE_BITS (10) +struct InterruptContext { + uint32_t cr8, cr2, ds; + uint8_t fxsave[512 + 16]; + uint32_t ebp, edi, esi, edx, ecx, ebx, eax; + uint32_t fromRing0, irq, errorCode; + uint32_t eip, cs, flags, esp, ss; + + // Note that when ring 0 is interrupted the order is different: + // esp, ss, irq, errorCode, eip, cs, flags. + // However we fix it before and after interrupts in InterruptHandler. +}; + +volatile uintptr_t tlbShootdownVirtualAddress; +volatile size_t tlbShootdownPageCount; +volatile size_t tlbShootdownProcessorsRemaining; + +#include #include - -size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi, int processorID) { - (void) interrupt; - (void) nmi; - (void) processorID; - // TODO. - return 0; -} - -void ArchInitialise() { - // TODO. -} - -void ArchNextTimer(size_t ms) { - // TODO. -} - -uint64_t ArchGetTimeMs() { - // TODO. - return 0; -} +#include InterruptContext *ArchInitialiseThread(uintptr_t kernelStack, uintptr_t kernelStackSize, Thread *thread, uintptr_t startAddress, uintptr_t argument1, uintptr_t argument2, bool userland, uintptr_t stack, uintptr_t userStackSize) { - // TODO. - return nullptr; + InterruptContext *context = ((InterruptContext *) (kernelStack + kernelStackSize - 12)) - 1; + thread->kernelStack = kernelStack + kernelStackSize - 12; + + *((uintptr_t *) (kernelStack + kernelStackSize - 12)) = (uintptr_t) KThreadTerminate; + *((uintptr_t *) (kernelStack + kernelStackSize - 8)) = argument1; + *((uintptr_t *) (kernelStack + kernelStackSize - 4)) = argument2; + + context->fxsave[32] = 0x80; + context->fxsave[33] = 0x1F; + + if (userland) { + context->cs = 0x1B; + context->ds = 0x23; + context->ss = 0x23; + } else { + context->cs = 0x08; + context->ds = 0x10; + context->ss = 0x10; + context->fromRing0 = true; + } + + context->flags = 1 << 9; // Interrupt flag + context->eip = startAddress; + context->esp = stack + userStackSize; + + return context; } -void ArchSwitchContext(InterruptContext *context, MMArchVAS *virtualAddressSpace, uintptr_t threadKernelStack, - Thread *newThread, MMSpace *oldAddressSpace) { - // TODO. -} +void TLBShootdownCallback() { + uintptr_t page = tlbShootdownVirtualAddress; -EsError ArchApplyRelocation(uintptr_t type, uint8_t *buffer, uintptr_t offset, uintptr_t result) { - // TODO. - return ES_ERROR_UNSUPPORTED_FEATURE; -} - -bool MMArchMapPage(MMSpace *space, uintptr_t physicalAddress, uintptr_t virtualAddress, unsigned flags) { - // TODO. - return false; -} - -void MMArchUnmapPages(MMSpace *space, uintptr_t virtualAddressStart, uintptr_t pageCount, unsigned flags, size_t unmapMaximum, uintptr_t *resumePosition) { - // TODO. -} - -bool MMArchMakePageWritable(MMSpace *space, uintptr_t virtualAddress) { - // TODO. - return false; -} - -bool MMArchHandlePageFault(uintptr_t address, uint32_t flags) { - // TODO. - return false; + // TODO How should this be determined? +#define INVALIDATE_ALL_PAGES_THRESHOLD (1024) + if (tlbShootdownPageCount > INVALIDATE_ALL_PAGES_THRESHOLD) { + ProcessorInvalidateAllPages(); + } else { + for (uintptr_t i = 0; i < tlbShootdownPageCount; i++, page += K_PAGE_SIZE) { + ProcessorInvalidatePage(page); + } + } } void MMArchInvalidatePages(uintptr_t virtualAddressStart, uintptr_t pageCount) { - // TODO. + KSpinlockAcquire(&ipiLock); + tlbShootdownVirtualAddress = virtualAddressStart; + tlbShootdownPageCount = pageCount; + tlbShootdownProcessorsRemaining = KGetCPUCount(); + + if (tlbShootdownProcessorsRemaining > 1) { + size_t ignored = ProcessorSendIPI(TLB_SHOOTDOWN_IPI); + __sync_fetch_and_sub(&tlbShootdownProcessorsRemaining, ignored); + while (tlbShootdownProcessorsRemaining); + } + + TLBShootdownCallback(); + KSpinlockRelease(&ipiLock); } bool MMArchIsBufferInUserRange(uintptr_t baseAddress, size_t byteCount) { // TODO. + KernelPanic("Unimplemented!\n"); return false; } bool MMArchSafeCopy(uintptr_t destinationAddress, uintptr_t sourceAddress, size_t byteCount) { // TODO. + KernelPanic("Unimplemented!\n"); return false; } bool MMArchCommitPageTables(MMSpace *space, MMRegion *region) { - // TODO. - return false; + KMutexAssertLocked(&space->reserveMutex); + + MMArchVAS *data = &space->data; + + uintptr_t base = region->baseAddress - (space == coreMMSpace ? MM_CORE_SPACE_START : 0); + uintptr_t end = base + (region->pageCount << K_PAGE_BITS); + uintptr_t needed = 0; + + for (uintptr_t i = base; i < end; i += 1L << (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)) { + uintptr_t indexL2 = i >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + if (!(data->l1Commit[indexL2 >> 3] & (1 << (indexL2 & 7)))) needed++; + i = indexL2 << (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + } + + if (needed) { + if (!MMCommit(needed * K_PAGE_SIZE, true)) { + return false; + } + + data->pageTablesCommitted += needed; + } + + for (uintptr_t i = base; i < end; i += 1L << (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)) { + uintptr_t indexL2 = i >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + data->l1Commit[indexL2 >> 3] |= (1 << (indexL2 & 7)); + i = indexL2 << (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + } + + return true; } bool MMArchInitialiseUserSpace(MMSpace *space, MMRegion *firstRegion) { // TODO. + KernelPanic("Unimplemented!\n"); return false; } -void MMArchInitialise() { - // TODO. -} - void MMArchFreeVAS(MMSpace *space) { // TODO. + KernelPanic("Unimplemented!\n"); } void MMArchFinalizeVAS(MMSpace *space) { // TODO. -} - -uintptr_t MMArchEarlyAllocatePage() { - // TODO. - return 0; -} - -uint64_t MMArchPopulatePageFrameDatabase() { - // TODO. - return 0; -} - -uintptr_t MMArchGetPhysicalMemoryHighest() { - // TODO. - return 0; -} - -void ProcessorDisableInterrupts() { - // TODO. -} - -void ProcessorEnableInterrupts() { - // TODO. -} - -bool ProcessorAreInterruptsEnabled() { - // TODO. - return false; -} - -void ProcessorHalt() { - // TODO. -} - -void ProcessorSendYieldIPI(Thread *thread) { - // TODO. -} - -void ProcessorFakeTimerInterrupt() { - // TODO. -} - -void ProcessorInvalidatePage(uintptr_t virtualAddress) { - // TODO. -} - -void ProcessorInvalidateAllPages() { - // TODO. -} - -void ProcessorFlushCodeCache() { - // TODO. -} - -void ProcessorFlushCache() { - // TODO. -} - -void ProcessorSetLocalStorage(CPULocalStorage *cls) { - // TODO. -} - -void ProcessorSetThreadStorage(uintptr_t tls) { - // TODO. -} - -void ProcessorSetAddressSpace(MMArchVAS *virtualAddressSpace) { - // TODO. -} - -uint64_t ProcessorReadTimeStamp() { - // TODO. - return 0; -} - -CPULocalStorage *GetLocalStorage() { - // TODO. - return nullptr; -} - -Thread *GetCurrentThread() { - // TODO. - return nullptr; -} - -uintptr_t MMArchTranslateAddress(MMSpace *space, uintptr_t virtualAddress, bool writeAccess) { - // TODO. - return 0; -} - -uint32_t KPCIReadConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, int size) { - // TODO. - return 0; -} - -void KPCIWriteConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value, int size) { - // TODO. -} - -bool KRegisterIRQ(intptr_t interruptIndex, KIRQHandler handler, void *context, const char *cOwnerName, KPCIDevice *pciDevice) { - // TODO. - return false; -} - -KMSIInformation KRegisterMSI(KIRQHandler handler, void *context, const char *cOwnerName) { - // TODO. - return {}; -} - -void KUnregisterMSI(uintptr_t tag) { - // TODO. -} - -void ProcessorOut8(uint16_t port, uint8_t value) { - // TODO. -} - -uint8_t ProcessorIn8(uint16_t port) { - // TODO. - return 0; -} - -void ProcessorOut16(uint16_t port, uint16_t value) { - // TODO. -} - -uint16_t ProcessorIn16(uint16_t port) { - // TODO. - return 0; -} - -void ProcessorOut32(uint16_t port, uint32_t value) { - // TODO. -} - -uint32_t ProcessorIn32(uint16_t port) { - // TODO. - return 0; -} - -void ProcessorReset() { - // TODO. -} - -uintptr_t ArchFindRootSystemDescriptorPointer() { - // TODO. - return 0; + KernelPanic("Unimplemented!\n"); } void ArchStartupApplicationProcessors() { // TODO. } -uintptr_t GetBootloaderInformationOffset() { - // TODO. - return 0; +bool MMArchHandlePageFault(uintptr_t address, uint32_t flags) { + address &= ~(K_PAGE_SIZE - 1); + bool forSupervisor = flags & MM_HANDLE_PAGE_FAULT_FOR_SUPERVISOR; + + if (!ProcessorAreInterruptsEnabled()) { + KernelPanic("MMArchHandlePageFault - Page fault with interrupts disabled.\n"); + } + + if (address >= MM_CORE_REGIONS_START && address < MM_CORE_REGIONS_START + MM_CORE_REGIONS_COUNT * sizeof(MMRegion) && forSupervisor) { + // This is where coreMMSpace stores its regions. + // Allocate physical memory and map it. + MMArchMapPage(kernelMMSpace, MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_ZEROED), address, MM_MAP_PAGE_COMMIT_TABLES_NOW); + return true; + } else if (address >= MM_CORE_SPACE_START && address < MM_CORE_SPACE_START + MM_CORE_SPACE_SIZE && forSupervisor) { + return MMHandlePageFault(coreMMSpace, address, flags); + } else if (address >= MM_KERNEL_SPACE_START && address < MM_KERNEL_SPACE_START + MM_KERNEL_SPACE_SIZE && forSupervisor) { + return MMHandlePageFault(kernelMMSpace, address, flags); + } else if (address >= MM_MODULES_START && address < MM_MODULES_START + MM_MODULES_SIZE && forSupervisor) { + return MMHandlePageFault(kernelMMSpace, address, flags); + } else if (address >= K_PAGE_SIZE) { + Thread *thread = GetCurrentThread(); + return MMHandlePageFault(thread->temporaryAddressSpace ?: thread->process->vmm, address, flags); + } + + return false; } -extern "C" void ProcessorDebugOutputByte(uint8_t byte) { - // TODO. +void ContextSanityCheck(InterruptContext *context) { + if (context->cs > 0x100 || context->ds > 0x100 || context->ss > 0x100 + || (context->eip < 0xC0000000 && context->cs == 0x08)) { + KernelPanic("ContextSanityCheck - Corrupt context (%x/%x/%x/%x/%x/%x)\n", + context, context->cs, context->ds, context->ss, context->eip, context->esp); + } +} + +extern "C" void InterruptHandler(InterruptContext *context) { + if (context->fromRing0) { + uint32_t esp = context->irq; + uint32_t ss = context->errorCode; + context->irq = context->eip; + context->errorCode = context->cs; + context->eip = context->flags; + context->cs = context->esp; + context->flags = context->ss; + context->esp = esp; + context->ss = ss; + } + + CPULocalStorage *local = GetLocalStorage(); + + if (scheduler.panic && context->irq != 2) { + goto end; + } + + if (context->irq < 0x20) { + // Processor exception. + + ProcessorEnableInterrupts(); + + if ((context->cs & 3) == 0) { + bool handled = false; + + if (context->irq == 0x0E && (~context->errorCode & (1 << 3))) { + if (MMArchHandlePageFault(context->cr2, MM_HANDLE_PAGE_FAULT_FOR_SUPERVISOR + | ((context->errorCode & 2) ? MM_HANDLE_PAGE_FAULT_WRITE : 0))) { + handled = true; + } + } + + if (!handled) { + KernelPanic("Unresolvable processor exception encountered in supervisor mode.\n%z\nEIP = %x (CPU %d)\nX86 error codes: [err] %x, [cr2] %x\n" + "Stack: [esp] %x, [ebp] %x\nRegisters: [eax] %x, [ebx] %x, [esi] %x, [edi] %x.\nThread ID = %d\n", + exceptionInformation[context->irq], context->eip, local ? local->processorID : -1, context->errorCode, context->cr2, + context->esp, context->ebp, context->eax, context->ebx, context->esi, context->edi, + local && local->currentThread ? local->currentThread->id : -1); + } + } else { + // TODO User exceptions. + } + } else if (context->irq == 0xFF) { + // Spurious interrupt (APIC), ignore. + } else if (context->irq >= 0x20 && context->irq < 0x30) { + // Spurious interrupt (PIC), ignore. + } else if (context->irq == TLB_SHOOTDOWN_IPI) { + TLBShootdownCallback(); + if (!tlbShootdownProcessorsRemaining) KernelPanic("InterruptHandler - tlbShootdownProcessorsRemaining is 0.\n"); + __sync_fetch_and_sub(&tlbShootdownProcessorsRemaining, 1); + } else if (context->irq == TIMER_INTERRUPT) { + if (local && scheduler.started && local->schedulerReady) { + scheduler.Yield(context); + } + } else if (context->irq >= INTERRUPT_VECTOR_MSI_START && context->irq < INTERRUPT_VECTOR_MSI_START + INTERRUPT_VECTOR_MSI_COUNT && local) { + KSpinlockAcquire(&irqHandlersLock); + MSIHandler handler = msiHandlers[context->irq - INTERRUPT_VECTOR_MSI_START]; + KSpinlockRelease(&irqHandlersLock); + local->irqSwitchThread = false; + + if (!handler.callback) { + KernelLog(LOG_ERROR, "Arch", "unexpected MSI", "Unexpected MSI vector %X (no handler).\n", context->irq); + } else { + handler.callback(context->irq - INTERRUPT_VECTOR_MSI_START, handler.context); + } + + if (local->irqSwitchThread && scheduler.started && local->schedulerReady) { + scheduler.Yield(context); // LapicEndOfInterrupt is called in PostContextSwitch. + } + + LapicEndOfInterrupt(); + } else if (context->irq >= IRQ_BASE && context->irq < IRQ_BASE + 0x20 && local) { + // See InterruptHandler in arch/x86_64/kernel.cpp for a discussion of what this is doing. + + local->irqSwitchThread = false; + local->inIRQ = true; + + uintptr_t line = context->irq - IRQ_BASE; + KSpinlockAcquire(&irqHandlersLock); + + for (uintptr_t i = 0; i < sizeof(irqHandlers) / sizeof(irqHandlers[0]); i++) { + IRQHandler handler = irqHandlers[i]; + if (!handler.callback) continue; + + if (handler.line == -1) { + if (line != 9 && line != 10 && line != 11) { + continue; + } else { + uint8_t mappedLine = pciIRQLines[handler.pciDevice->slot][handler.pciDevice->interruptPin - 1]; + + if (mappedLine && line != mappedLine) { + continue; + } + } + } else { + if ((uintptr_t) handler.line != line) { + continue; + } + } + + KSpinlockRelease(&irqHandlersLock); + handler.callback(context->irq - IRQ_BASE, handler.context); + KSpinlockAcquire(&irqHandlersLock); + } + + KSpinlockRelease(&irqHandlersLock); + local->inIRQ = false; + + if (local->irqSwitchThread && scheduler.started && local->schedulerReady) { + scheduler.Yield(context); + } + + LapicEndOfInterrupt(); + } + + if (context->irq >= 0x30 && context->irq != 0xFF) { + LapicEndOfInterrupt(); + } + + end:; + + ContextSanityCheck(context); + + if (context->fromRing0) { + uint32_t irq = context->esp; + uint32_t errorCode = context->ss; + context->ss = context->flags; + context->esp = context->cs; + context->flags = context->eip; + context->cs = context->errorCode; + context->eip = context->irq; + context->irq = irq; + context->errorCode = errorCode; + } } #include diff --git a/arch/x86_32/kernel.s b/arch/x86_32/kernel.s index 9f5cb65..05feeba 100644 --- a/arch/x86_32/kernel.s +++ b/arch/x86_32/kernel.s @@ -1,8 +1,553 @@ [bits 32] +[global ProcessorReset] +[global ProcessorDebugOutputByte] +[global ProcessorIn16] +[global ProcessorIn32] +[global ProcessorIn8] +[global ProcessorOut16] +[global ProcessorOut32] +[global ProcessorOut8] +[global ProcessorHalt] +[global ProcessorDisableInterrupts] +[global ProcessorEnableInterrupts] +[global ProcessorAreInterruptsEnabled] +[global ProcessorSetLocalStorage] +[global ProcessorReadCR3] +[global ProcessorInvalidatePage] +[global ProcessorInvalidateAllPages] +[global ProcessorReadTimeStamp] +[global ProcessorSetThreadStorage] +[global ProcessorFakeTimerInterrupt] +[global ProcessorSetAddressSpace] +[global ProcessorFlushCodeCache] +[global GetLocalStorage] +[global GetCurrentThread] +[global ArchSwitchContext] +[global processorGDTR] +[global timeStampCounterSynchronizationValue] + +[extern ArchNextTimer] +[extern InterruptHandler] +[extern KThreadTerminate] +[extern KernelMain] +[extern PostContextSwitch] +[extern SetupProcessor2] +[extern Syscall] +[extern installationID] +[extern PCSetupCOM1] +[extern PCDisablePIC] +[extern PCProcessMemoryMap] +[extern bootloaderID] +[extern bootloaderInformationOffset] + +[section .bss] + +%define boot_stack_size 10000 +boot_stack: resb boot_stack_size + +%define idt_size 2048 +idt_data: resb idt_size + +%define cpu_local_storage_size (256 * 4 * 4) +cpu_local_storage: resb cpu_local_storage_size +cpu_local_index: resb 4 + +timeStampCounterSynchronizationValue: resb 8 + +[section .data] + +processorGDTR: + times 8 db 0 + +idt: + .limit: dw idt_size - 1 + .base: dd idt_data + align 4 + +gdt_data: + .null_entry: dq 0 + .code_entry: dd 0xFFFF ; 0x0008 + db 0 + dw 0xCF9A + db 0 + .data_entry: dd 0xFFFF ; 0x0010 + db 0 + dw 0xCF92 + db 0 + .user_code: dd 0xFFFF ; 0x001B + db 0 + dw 0xCFFA + db 0 + .user_data: dd 0xFFFF ; 0x0023 + db 0 + dw 0xCFF2 + db 0 + .tss: dd 0x68 ; 0x0028 + db 0 + dw 0xE9 + db 0 + dq 0 + .local: times 256 dq 0 ; 0x0038 - 0x0838 + .gdt: dw (gdt_data.gdt - gdt_data - 1) + .gdt2: dd gdt_data + [section .text] [global _start] _start: - ; TODO - jmp $ + ; Save the bootloader ID and information offset. + mov [bootloaderID],esi + mov [bootloaderInformationOffset],edi + + ; The MBR bootloader does not know the address of the RSDP. + cmp edi,0 + jne .standard_acpi + mov [0x7FE8],edi + .standard_acpi: + + ; Install the boot stack. + mov esp,boot_stack + boot_stack_size + + ; Load the installation ID. + mov eax,[edi + 0x7FF0] + mov [installationID + 0],eax + mov eax,[edi + 0x7FF4] + mov [installationID + 4],eax + mov eax,[edi + 0x7FF8] + mov [installationID + 8],eax + mov eax,[edi + 0x7FFC] + mov [installationID + 12],eax + + ; Load the new GDT, saving the location of the bootstrap GDT. + lgdt [gdt_data.gdt] + sgdt [processorGDTR] + + ; Move the identity paging the bootloader used to LOW_MEMORY_MAP_START. + ; Then, map the local APIC to LOCAL_APIC_BASE. + mov eax,[0xFFFFF000] + mov [0xFFFFFEC0],eax + xor eax,eax + mov [0xFFFFF000],eax + mov eax,0xFEE00103 + mov [0xFFC00000 + (0xEC3FF << 2)],eax + mov eax,cr3 + mov cr3,eax + + ; Install the interrupt handlers + mov ebx,idt_data +%macro INSTALL_INTERRUPT_HANDLER 1 + mov edx,InterruptHandler%1 + call InstallInterruptHandler + add ebx,8 +%endmacro +%assign i 0 +%rep 256 + INSTALL_INTERRUPT_HANDLER i +%assign i i+1 +%endrep + + ; Setup the remaining things and call KernelMain. + call SetupProcessor1 ; Need to get SSE up before calling into C code. + call PCSetupCOM1 + call PCDisablePIC + call PCProcessMemoryMap + call KernelMain + + ; Fall-through. +ProcessorReady: + ; Set the timer and become this CPU's idle thread. + push 1 + call ArchNextTimer + + ; Fall-through. +ProcessorIdle: + sti + hlt + jmp ProcessorIdle + +SetupProcessor1: + ; x87 FPU. + fninit + fldcw [.cw] + jmp .cwa + .cw: dw 0x037A + .cwa: + + ; Enable MMX, SSE and SSE2. + ; TODO Check these are actually present! + mov eax,cr0 + mov ebx,cr4 + and eax,~4 + or eax,2 + or ebx,512 + 1024 + mov cr0,eax + mov cr4,ebx + + ; Setup the local storage. + ; This creates a new data segment in the GDT pointing to a unique 16-byte block of cpu_local_storage, + ; and updates FS to use the data segment. + mov eax,[cpu_local_index] + mov ebx,eax + shl ebx,4 + add ebx,cpu_local_storage + mov edx,ebx + shl ebx,16 + or ebx,0x0000FFFF + mov ecx,edx + shr ecx,16 + and edx,0xFF000000 + or edx,0x00CF9200 + or dl,cl + mov dword [gdt_data.local + eax * 8 + 0],ebx + mov dword [gdt_data.local + eax * 8 + 4],edx + lea eax,[0x0038 + eax * 8] + mov fs,ax + inc dword [cpu_local_index] + + ; Enable global pages. + mov eax,cr4 + or eax,1 << 7 + mov cr4,eax + + ; Enable write protect, so copy-on-write works in the kernel, and MMArchSafeCopy will page fault in read-only regions. + mov eax,cr0 + or eax,1 << 16 + mov cr0,eax + + ; Load the IDTR. + lidt [idt] + sti + + ; Enable the APIC. + ; TODO Check it is actually present! + mov ecx,0x1B + rdmsr + or eax,0x800 + wrmsr + + ; Set the spurious interrupt vector to 0xFF + mov eax,0xEC3FF0F0 + mov ebx,[eax] + or ebx,0x1FF + mov [eax],ebx + + ; Use the flat processor addressing model + mov eax,0xEC3FF0E0 + mov dword [eax],0xFFFFFFFF + + ; Make sure that no external interrupts are masked + xor eax,eax + mov [0xEC3FF080],eax + + ; TODO More feature detection and initialisation! + + ret + +InstallInterruptHandler: + mov word [ebx + 0],dx + mov word [ebx + 2],0x0008 + mov word [ebx + 4],0x8E00 + shr edx,16 + mov word [ebx + 6],dx + ret + +%macro INTERRUPT_HANDLER 1 +InterruptHandler%1: + push dword 0 ; A fake error code + push dword %1 ; The interrupt number + jmp ASMInterruptHandler +%endmacro + +%macro INTERRUPT_HANDLER_EC 1 +InterruptHandler%1: + ; The CPU already pushed an error code + push dword %1 ; The interrupt number + jmp ASMInterruptHandler +%endmacro + +INTERRUPT_HANDLER 0 +INTERRUPT_HANDLER 1 +INTERRUPT_HANDLER 2 +INTERRUPT_HANDLER 3 +INTERRUPT_HANDLER 4 +INTERRUPT_HANDLER 5 +INTERRUPT_HANDLER 6 +INTERRUPT_HANDLER 7 +INTERRUPT_HANDLER_EC 8 +INTERRUPT_HANDLER 9 +INTERRUPT_HANDLER_EC 10 +INTERRUPT_HANDLER_EC 11 +INTERRUPT_HANDLER_EC 12 +INTERRUPT_HANDLER_EC 13 +INTERRUPT_HANDLER_EC 14 +INTERRUPT_HANDLER 15 +INTERRUPT_HANDLER 16 +INTERRUPT_HANDLER_EC 17 +INTERRUPT_HANDLER 18 +INTERRUPT_HANDLER 19 +INTERRUPT_HANDLER 20 +INTERRUPT_HANDLER 21 +INTERRUPT_HANDLER 22 +INTERRUPT_HANDLER 23 +INTERRUPT_HANDLER 24 +INTERRUPT_HANDLER 25 +INTERRUPT_HANDLER 26 +INTERRUPT_HANDLER 27 +INTERRUPT_HANDLER 28 +INTERRUPT_HANDLER 29 +INTERRUPT_HANDLER 30 +INTERRUPT_HANDLER 31 + +%assign i 32 +%rep 224 +INTERRUPT_HANDLER i +%assign i i+1 +%endrep + +ASMInterruptHandler: + cld + + test byte [esp + 12],3 + jnz .have_esp + ; When ring 0 is interrupted, ESP and SS aren't pushed. + ; We push them ourselves here; we'll fix the order later. + push eax + push esp + mov eax,ss + xchg [esp + 4],eax + push 1 + jmp .fixed + .have_esp: + push 0 + .fixed: + + push eax + push ebx + push ecx + push edx + push esi + push edi + push ebp + + mov ebx,esp + and esp,~0xF + fxsave [esp - 512] + mov esp,ebx + sub esp,512 + 16 + + mov eax,ds + push eax + mov eax,0x10 + mov ds,ax + mov eax,cr2 + push eax + + mov edx,[0xEC3FF080] + push edx + mov edx,0xF0 + mov [0xEC3FF080],edx ; Mask all interrupts. + sti ; ...so there's no need to have the interrupt flag clear. + + push esp + call InterruptHandler + add esp,4 + xor eax,eax + + .return: + + cli ; Must be done before restoring CR8. + pop edx + mov [0xEC3FF080],edx + add esp,4 + pop ebx + mov ds,bx + + add esp,512 + 16 + mov ebx,esp + and ebx,~0xF + fxrstor [ebx - 512] + + or al,al + jz .old_thread + fninit ; New thread - initialise FPU. + .old_thread: + + pop ebp + pop edi + pop esi + pop edx + pop ecx + pop ebx + pop eax + + test byte [esp],1 + jz .need_esp + ; When returning to ring 0, ESP and SS aren't popped. + ; So we do it manually here. + add esp,8 + .need_esp: + + add esp,12 + iret + +ArchSwitchContext: + cli + mov eax,[esp + 16] + mov [fs:8],eax + mov ebx,[esp + 12] + mov [fs:4],ebx + mov edi,[esp + 8] + mov ecx,[edi] + mov edx,cr3 + cmp edx,ecx + je .cont + mov cr3,ecx + .cont: + mov eax,[esp + 4] + mov ecx,[esp + 20] + mov esp,eax ; Put the stack just below the context. + push ecx + push eax + call PostContextSwitch + add esp,8 + jmp ASMInterruptHandler.return + +ProcessorDebugOutputByte: +%ifdef COM_OUTPUT + mov dx,0x3F8 + 5 + .WaitRead: + in al,dx + and al,0x20 + cmp al,0 + je .WaitRead + mov dx,0x3F8 + 0 + mov eax,[esp + 4] + out dx,al +%endif + ret + +ProcessorOut8: + mov edx,[esp + 4] + mov eax,[esp + 8] + out dx,al + ret + +ProcessorIn8: + mov edx,[esp + 4] + xor eax,eax + in al,dx + ret + +ProcessorOut16: + mov edx,[esp + 4] + mov eax,[esp + 8] + out dx,ax + ret + +ProcessorIn16: + mov edx,[esp + 4] + xor eax,eax + in ax,dx + ret + +ProcessorOut32: + mov edx,[esp + 4] + mov eax,[esp + 8] + out dx,eax + ret + +ProcessorIn32: + mov edx,[esp + 4] + xor eax,eax + in eax,dx + ret + +ProcessorReset: + in al,0x64 + test al,2 + jne ProcessorReset + mov al,0xFE + out 0x64,al + + ; Fall-through. +ProcessorHalt: + cli + hlt + jmp ProcessorHalt + +ProcessorDisableInterrupts: + mov eax,0xE0 ; Still allow important IPIs to go through. + mov [0xEC3FF080],eax + ret + +ProcessorEnableInterrupts: + xor eax,eax + mov [0xEC3FF080],eax + ret + +ProcessorAreInterruptsEnabled: + xor al,al + mov edx,[0xEC3FF080] + or edx,edx + jnz .done + mov al,1 + .done: + ret + +GetLocalStorage: + mov eax,[fs:0] + ret + +GetCurrentThread: + mov eax,[fs:8] + ret + +ProcessorSetLocalStorage: + mov eax,[esp + 4] + mov [fs:0],eax + ret + +ProcessorReadCR3: + mov eax,cr3 + ret + +ProcessorInvalidatePage: + mov eax,[esp + 4] + invlpg [eax] + ret + +ProcessorInvalidateAllPages: + ; Toggle CR4.PGE to invalidate all TLB entries, including global entries. + mov eax,cr4 + and eax,~(1 << 7) + mov cr4,eax + or eax,1 << 7 + mov cr4,eax + ret + +ProcessorReadTimeStamp: + rdtsc + ret + +ProcessorSetThreadStorage: + ; TODO. + ret + +ProcessorFakeTimerInterrupt: + int 0x40 + ret + +ProcessorSetAddressSpace: + mov eax,[esp + 4] + mov edx,[eax] + mov eax,cr3 + cmp eax,edx + je .cont + mov cr3,edx + .cont: + ret + +ProcessorFlushCodeCache: + wbinvd + ret diff --git a/arch/x86_64.h b/arch/x86_64.h deleted file mode 100644 index cec3218..0000000 --- a/arch/x86_64.h +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef ARCH_X86_64_HEADER -#define ARCH_X86_64_HEADER - -#ifndef ES_ARCH_X86_64 -#error Included x86_64.h but not targeting x86_64. -#endif - -#include "x86_pc.h" - -// --------------------------------- Interrupt vectors. - -// Interrupt vectors: -// 0x00 - 0x1F: CPU exceptions -// 0x20 - 0x2F: PIC (disabled, spurious) -// 0x30 - 0x4F: Timers and low-priority IPIs. -// 0x50 - 0x6F: APIC (standard) -// 0x70 - 0xAF: MSI -// 0xF0 - 0xFE: High-priority IPIs -// 0xFF: APIC (spurious interrupt) - -#define TIMER_INTERRUPT (0x40) -#define YIELD_IPI (0x41) -#define IRQ_BASE (0x50) -#define CALL_FUNCTION_ON_ALL_PROCESSORS_IPI (0xF0) -#define KERNEL_PANIC_IPI (0) // NMIs ignore the interrupt vector. - -#define INTERRUPT_VECTOR_MSI_START (0x70) -#define INTERRUPT_VECTOR_MSI_COUNT (0x40) - -// --------------------------------- Forward declarations. - -extern "C" void gdt_data(); -extern "C" void processorGDTR(); - -extern "C" uint64_t ProcessorReadCR3(); -extern "C" uintptr_t ProcessorGetRSP(); -extern "C" uintptr_t ProcessorGetRBP(); -extern "C" uint64_t ProcessorReadMXCSR(); -extern "C" void ProcessorInstallTSS(uint32_t *gdt, uint32_t *tss); -extern "C" void ProcessorAPStartup(); -extern "C" void ProcessorReset(); - -extern "C" void SSSE3Framebuffer32To24Copy(volatile uint8_t *destination, volatile uint8_t *source, size_t pixelGroups); -extern "C" uintptr_t _KThreadTerminate; - -extern bool pagingNXESupport; -extern bool pagingPCIDSupport; -extern bool pagingSMEPSupport; -extern bool pagingTCESupport; - -extern volatile uint64_t timeStampCounterSynchronizationValue; - -extern "C" bool simdSSE3Support; -extern "C" bool simdSSSE3Support; - -extern uintptr_t bootloaderInformationOffset; - -struct NewProcessorStorage { - struct CPULocalStorage *local; - uint32_t *gdt; -}; - -NewProcessorStorage AllocateNewProcessorStorage(struct ArchCPU *archCPU); -extern "C" void SetupProcessor2(struct NewProcessorStorage *); -void ArchDelay1Ms(); // Spin for approximately 1ms. Use only during initialisation. Not thread-safe. -uint64_t ArchGetTimeFromPITMs(); -void *ACPIGetRSDP(); -size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi = false, int processorID = -1); // Returns the number of processors the IPI was *not* sent to. -void ArchSetPCIIRQLine(uint8_t slot, uint8_t pin, uint8_t line); -uintptr_t ArchFindRootSystemDescriptorPointer(); -void ArchStartupApplicationProcessors(); - -struct InterruptContext { - uint64_t cr2, ds; - uint8_t fxsave[512 + 16]; - uint64_t _check, cr8; - uint64_t r15, r14, r13, r12, r11, r10, r9, r8; - uint64_t rbp, rdi, rsi, rdx, rcx, rbx, rax; - uint64_t interruptNumber, errorCode; - uint64_t rip, cs, flags, rsp, ss; -}; - -#endif diff --git a/arch/x86_64/kernel.cpp b/arch/x86_64/kernel.cpp index 73448e5..06695fe 100644 --- a/arch/x86_64/kernel.cpp +++ b/arch/x86_64/kernel.cpp @@ -45,52 +45,12 @@ struct MMArchVAS { #ifdef IMPLEMENTATION -#include -#include - #define MM_CORE_SPACE_START (0xFFFF800100000000) #define MM_CORE_SPACE_SIZE (0xFFFF8001F0000000 - 0xFFFF800100000000) #define MM_USER_SPACE_START (0x100000000000) #define MM_USER_SPACE_SIZE (0xF00000000000 - 0x100000000000) -#define LOW_MEMORY_MAP_START (0xFFFFFE0000000000) -#define LOW_MEMORY_LIMIT (0x100000000) // The first 4GB is mapped here. - -struct MSIHandler { - KIRQHandler callback; - void *context; -}; - -struct IRQHandler { - KIRQHandler callback; - void *context; - intptr_t line; - KPCIDevice *pciDevice; - const char *cOwnerName; -}; - -extern PhysicalMemoryRegion *physicalMemoryRegions; -extern size_t physicalMemoryRegionsCount; -extern size_t physicalMemoryRegionsPagesCount; -extern size_t physicalMemoryOriginalPagesCount; -extern size_t physicalMemoryRegionsIndex; -extern uintptr_t physicalMemoryHighest; - -uint8_t pciIRQLines[0x100 /* slots */][4 /* pins */]; - -MSIHandler msiHandlers[INTERRUPT_VECTOR_MSI_COUNT]; -IRQHandler irqHandlers[0x40]; -KSpinlock irqHandlersLock; // Also for msiHandlers. - -KSpinlock ipiLock; - -uint8_t coreL1Commit[(0xFFFF800200000000 - 0xFFFF800100000000) >> (/* ENTRIES_PER_PAGE_TABLE_BITS */ 9 + K_PAGE_BITS + 3)]; - -volatile uintptr_t tlbShootdownVirtualAddress; -volatile size_t tlbShootdownPageCount; - -typedef void (*CallFunctionOnAllProcessorsCallbackFunction)(); -volatile CallFunctionOnAllProcessorsCallbackFunction callFunctionOnAllProcessorsCallback; -volatile uintptr_t callFunctionOnAllProcessorsRemaining; +#define LOW_MEMORY_MAP_START (0xFFFFFE0000000000) +#define LOW_MEMORY_LIMIT (0x100000000) // The first 4GB is mapped here. // Recursive page table mapping in slot 0x1FE, so that the top 2GB are available for mcmodel kernel. #define PAGE_TABLE_L4 ((volatile uint64_t *) 0xFFFFFF7FBFDFE000) @@ -100,22 +60,49 @@ volatile uintptr_t callFunctionOnAllProcessorsRemaining; #define ENTRIES_PER_PAGE_TABLE (512) #define ENTRIES_PER_PAGE_TABLE_BITS (9) -uint32_t LapicReadRegister(uint32_t reg) { - return acpi.lapicAddress[reg]; -} +uint8_t coreL1Commit[(0xFFFF800200000000 - 0xFFFF800100000000) >> (/* ENTRIES_PER_PAGE_TABLE_BITS */ 9 + K_PAGE_BITS + 3)]; -void LapicWriteRegister(uint32_t reg, uint32_t value) { - acpi.lapicAddress[reg] = value; -} +extern "C" void gdt_data(); +extern "C" void processorGDTR(); -void LapicNextTimer(size_t ms) { - LapicWriteRegister(0x320 >> 2, TIMER_INTERRUPT | (1 << 17)); - LapicWriteRegister(0x380 >> 2, acpi.lapicTicksPerMs * ms); -} +extern "C" uint64_t ProcessorReadCR3(); +extern "C" uintptr_t ProcessorGetRSP(); +extern "C" uintptr_t ProcessorGetRBP(); +extern "C" uint64_t ProcessorReadMXCSR(); +extern "C" void ProcessorInstallTSS(uint32_t *gdt, uint32_t *tss); +extern "C" void ProcessorAPStartup(); -void LapicEndOfInterrupt() { - LapicWriteRegister(0xB0 >> 2, 0); -} +extern "C" void SSSE3Framebuffer32To24Copy(volatile uint8_t *destination, volatile uint8_t *source, size_t pixelGroups); +extern "C" uintptr_t _KThreadTerminate; + +extern bool pagingNXESupport; +extern bool pagingPCIDSupport; +extern bool pagingSMEPSupport; +extern bool pagingTCESupport; + +extern "C" bool simdSSE3Support; +extern "C" bool simdSSSE3Support; + +struct InterruptContext { + uint64_t cr2, ds; + uint8_t fxsave[512 + 16]; + uint64_t _check, cr8; + uint64_t r15, r14, r13, r12, r11, r10, r9, r8; + uint64_t rbp, rdi, rsi, rdx, rcx, rbx, rax; + uint64_t interruptNumber, errorCode; + uint64_t rip, cs, flags, rsp, ss; +}; + +#include +#include +#include + +volatile uintptr_t tlbShootdownVirtualAddress; +volatile size_t tlbShootdownPageCount; + +typedef void (*CallFunctionOnAllProcessorsCallbackFunction)(); +volatile CallFunctionOnAllProcessorsCallbackFunction callFunctionOnAllProcessorsCallback; +volatile uintptr_t callFunctionOnAllProcessorsRemaining; void ArchSetPCIIRQLine(uint8_t slot, uint8_t pin, uint8_t line) { pciIRQLines[slot][pin] = line; @@ -184,157 +171,6 @@ bool MMArchCommitPageTables(MMSpace *space, MMRegion *region) { return true; } -bool MMArchMakePageWritable(MMSpace *space, uintptr_t virtualAddress) { - KMutexAcquire(&space->data.mutex); - EsDefer(KMutexRelease(&space->data.mutex)); - - virtualAddress &= 0x0000FFFFFFFFF000; - - uintptr_t indexL4 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3); - uintptr_t indexL3 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2); - uintptr_t indexL2 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); - uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); - - if ((PAGE_TABLE_L4[indexL4] & 1) == 0) return false; - if ((PAGE_TABLE_L3[indexL3] & 1) == 0) return false; - if ((PAGE_TABLE_L2[indexL2] & 1) == 0) return false; - if ((PAGE_TABLE_L1[indexL1] & 1) == 0) return false; - - PAGE_TABLE_L1[indexL1] |= 2; - return true; -} - -bool MMArchMapPage(MMSpace *space, uintptr_t physicalAddress, uintptr_t virtualAddress, unsigned flags) { - // TODO Use the no-execute bit. - - if (physicalAddress & (K_PAGE_SIZE - 1)) { - KernelPanic("MMArchMapPage - Physical address not page aligned.\n"); - } - - if (pmm.pageFrames && (physicalAddress >> K_PAGE_BITS) < pmm.pageFrameDatabaseCount) { - if (pmm.pageFrames[physicalAddress >> K_PAGE_BITS].state != MMPageFrame::ACTIVE - && pmm.pageFrames[physicalAddress >> K_PAGE_BITS].state != MMPageFrame::UNUSABLE) { - KernelPanic("MMArchMapPage - Physical page frame %x not marked as ACTIVE or UNUSABLE.\n", physicalAddress); - } - } - - bool acquireFrameLock = !(flags & (MM_MAP_PAGE_NO_NEW_TABLES | MM_MAP_PAGE_FRAME_LOCK_ACQUIRED)); - if (acquireFrameLock) KMutexAcquire(&pmm.pageFrameMutex); - EsDefer(if (acquireFrameLock) KMutexRelease(&pmm.pageFrameMutex);); - - bool acquireSpaceLock = ~flags & MM_MAP_PAGE_NO_NEW_TABLES; - if (acquireSpaceLock) KMutexAcquire(&space->data.mutex); - EsDefer(if (acquireSpaceLock) KMutexRelease(&space->data.mutex)); - - uintptr_t cr3 = space->data.cr3; - - if (virtualAddress < 0xFFFF800000000000 && ProcessorReadCR3() != cr3) { - KernelPanic("MMArchMapPage - Attempt to map page into other address space.\n"); - } else if (!physicalAddress) { - KernelPanic("MMArchMapPage - Attempt to map physical page 0.\n"); - } else if (!virtualAddress) { - KernelPanic("MMArchMapPage - Attempt to map virtual page 0.\n"); - } - - // EsPrint("\tMap, %x -> %x\n", virtualAddress, physicalAddress); - - uintptr_t oldVirtualAddress = virtualAddress; - physicalAddress &= 0xFFFFFFFFFFFFF000; - virtualAddress &= 0x0000FFFFFFFFF000; - - uintptr_t indexL4 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3); - uintptr_t indexL3 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2); - uintptr_t indexL2 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); - uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); - - if (space != coreMMSpace && space != kernelMMSpace /* Don't check the kernel's space since the bootloader's tables won't be committed. */) { - if (!(space->data.l3Commit[indexL4 >> 3] & (1 << (indexL4 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L3 page table.\n"); - if (!(space->data.l2Commit[indexL3 >> 3] & (1 << (indexL3 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L2 page table.\n"); - if (!(space->data.l1Commit[indexL2 >> 3] & (1 << (indexL2 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L1 page table.\n"); - } - - if ((PAGE_TABLE_L4[indexL4] & 1) == 0) { - if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); - PAGE_TABLE_L4[indexL4] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; - ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L3 + indexL3)); // Not strictly necessary. - EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L3 + indexL3) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); - space->data.pageTablesActive++; - } - - if ((PAGE_TABLE_L3[indexL3] & 1) == 0) { - if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); - PAGE_TABLE_L3[indexL3] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; - ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L2 + indexL2)); // Not strictly necessary. - EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L2 + indexL2) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); - space->data.pageTablesActive++; - } - - if ((PAGE_TABLE_L2[indexL2] & 1) == 0) { - if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); - PAGE_TABLE_L2[indexL2] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; - ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L1 + indexL1)); // Not strictly necessary. - EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L1 + indexL1) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); - space->data.pageTablesActive++; - } - - uintptr_t oldValue = PAGE_TABLE_L1[indexL1]; - uintptr_t value = physicalAddress | 3; - - if (flags & MM_MAP_PAGE_WRITE_COMBINING) value |= 16; // This only works because we modified the PAT in SetupProcessor1. - if (flags & MM_MAP_PAGE_NOT_CACHEABLE) value |= 24; - if (flags & MM_MAP_PAGE_USER) value |= 7; - else value |= 1 << 8; // Global. - if (flags & MM_MAP_PAGE_READ_ONLY) value &= ~2; - if (flags & MM_MAP_PAGE_COPIED) value |= 1 << 9; // Ignored by the CPU. - - // When the CPU accesses or writes to a page, - // it will modify the table entry to set the accessed or dirty bits respectively, - // but it uses its TLB entry as the assumed previous value of the entry. - // When unmapping pages we can't atomically remove an entry and do the TLB shootdown. - // This creates a race condition: - // 1. CPU 0 maps a page table entry. The dirty bit is not set. - // 2. CPU 1 reads from the page. A TLB entry is created with the dirty bit not set. - // 3. CPU 0 unmaps the entry. - // 4. CPU 1 writes to the page. As the TLB entry has the dirty bit cleared, it sets the entry to its cached entry ORed with the dirty bit. - // 5. CPU 0 invalidates the entry. - // That is, CPU 1 didn't realize the page was unmapped when it wrote out its entry, so the page becomes mapped again. - // To prevent this, we mark all pages with the dirty and accessed bits when we initially map them. - // (We don't use these bits for anything, anyway. They're basically useless on SMP systems, as far as I can tell.) - // That said, a CPU won't overwrite and clear a dirty bit when writing out its accessed flag (tested on Qemu); - // see here https://stackoverflow.com/questions/69024372/. - // Tl;dr: if a CPU ever sees an entry without these bits set, it can overwrite the entry with junk whenever it feels like it. - // TODO Should we be marking page tables as dirty/accessed? (Including those made by the 32-bit AND 64-bit bootloader and MMArchInitialise). - value |= (1 << 5) | (1 << 6); - - if ((oldValue & 1) && !(flags & MM_MAP_PAGE_OVERWRITE)) { - if (flags & MM_MAP_PAGE_IGNORE_IF_MAPPED) { - return false; - } - - if ((oldValue & ~(K_PAGE_SIZE - 1)) != physicalAddress) { - KernelPanic("MMArchMapPage - Attempt to map %x to %x that has already been mapped to %x.\n", - virtualAddress, physicalAddress, oldValue & (~(K_PAGE_SIZE - 1))); - } - - if (oldValue == value) { - KernelPanic("MMArchMapPage - Attempt to rewrite page translation.\n", - physicalAddress, virtualAddress, oldValue & (K_PAGE_SIZE - 1), value & (K_PAGE_SIZE - 1)); - } else if (!(oldValue & 2) && (value & 2)) { - // The page has become writable. - } else { - KernelPanic("MMArchMapPage - Attempt to change flags mapping %x address %x from %x to %x.\n", - physicalAddress, virtualAddress, oldValue & (K_PAGE_SIZE - 1), value & (K_PAGE_SIZE - 1)); - } - } - - PAGE_TABLE_L1[indexL1] = value; - - // We rely on this page being invalidated on this CPU in some places. - ProcessorInvalidatePage(oldVirtualAddress); - - return true; -} - bool MMArchIsBufferInUserRange(uintptr_t baseAddress, size_t byteCount) { if (baseAddress & 0xFFFF800000000000) return false; if (byteCount & 0xFFFF800000000000) return false; @@ -354,8 +190,7 @@ bool MMArchHandlePageFault(uintptr_t address, uint32_t flags) { if (address < K_PAGE_SIZE) { } else if (address >= LOW_MEMORY_MAP_START && address < LOW_MEMORY_MAP_START + LOW_MEMORY_LIMIT && forSupervisor) { // We want to access a physical page within the first 4GB. - // This is used for device IO, so the page can't be cacheable. - MMArchMapPage(kernelMMSpace, address - LOW_MEMORY_MAP_START, address, MM_MAP_PAGE_NOT_CACHEABLE | MM_MAP_PAGE_COMMIT_TABLES_NOW); + MMArchMapPage(kernelMMSpace, address - LOW_MEMORY_MAP_START, address, MM_MAP_PAGE_COMMIT_TABLES_NOW); return true; } else if (address >= MM_CORE_REGIONS_START && address < MM_CORE_REGIONS_START + MM_CORE_REGIONS_COUNT * sizeof(MMRegion) && forSupervisor) { // This is where coreMMSpace stores its regions. @@ -378,40 +213,6 @@ bool MMArchHandlePageFault(uintptr_t address, uint32_t flags) { return false; } -void MMArchInitialise() { - coreMMSpace->data.cr3 = kernelMMSpace->data.cr3 = ProcessorReadCR3(); - coreMMSpace->data.l1Commit = coreL1Commit; - - mmCoreRegions[0].baseAddress = MM_CORE_SPACE_START; - mmCoreRegions[0].pageCount = MM_CORE_SPACE_SIZE / K_PAGE_SIZE; - - for (uintptr_t i = 0x100; i < 0x200; i++) { - if (PAGE_TABLE_L4[i] == 0) { - // We don't need to commit anything because the PMM isn't ready yet. - PAGE_TABLE_L4[i] = MMPhysicalAllocate(ES_FLAGS_DEFAULT) | 3; - EsMemoryZero((void *) (PAGE_TABLE_L3 + i * 0x200), K_PAGE_SIZE); - } - } - - KMutexAcquire(&coreMMSpace->reserveMutex); - kernelMMSpace->data.l1Commit = (uint8_t *) MMReserve(coreMMSpace, L1_COMMIT_SIZE_BYTES, MM_REGION_NORMAL | MM_REGION_NO_COMMIT_TRACKING | MM_REGION_FIXED)->baseAddress; - KMutexRelease(&coreMMSpace->reserveMutex); -} - -uintptr_t MMArchTranslateAddress(MMSpace *, uintptr_t virtualAddress, bool writeAccess) { - // TODO I don't think this mutex was ever necessary? - // space->data.mutex.Acquire(); - // EsDefer(space->data.mutex.Release()); - - virtualAddress &= 0x0000FFFFFFFFF000; - if ((PAGE_TABLE_L4[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3)] & 1) == 0) return 0; - if ((PAGE_TABLE_L3[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2)] & 1) == 0) return 0; - if ((PAGE_TABLE_L2[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)] & 1) == 0) return 0; - uintptr_t physicalAddress = PAGE_TABLE_L1[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0)]; - if (writeAccess && !(physicalAddress & 2)) return 0; - return (physicalAddress & 1) ? (physicalAddress & 0x0000FFFFFFFFF000) : 0; -} - void ArchCallFunctionOnAllProcessors(CallFunctionOnAllProcessorsCallbackFunction callback, bool includingThisProcessor) { KSpinlockAssertLocked(&ipiLock); @@ -459,88 +260,6 @@ void MMArchInvalidatePages(uintptr_t virtualAddressStart, uintptr_t pageCount) { KSpinlockRelease(&ipiLock); } -void MMArchUnmapPages(MMSpace *space, uintptr_t virtualAddressStart, uintptr_t pageCount, unsigned flags, size_t unmapMaximum, uintptr_t *resumePosition) { - // We can't let anyone use the unmapped pages until they've been invalidated on all processors. - // This also synchronises modified bit updating. - KMutexAcquire(&pmm.pageFrameMutex); - EsDefer(KMutexRelease(&pmm.pageFrameMutex)); - - KMutexAcquire(&space->data.mutex); - EsDefer(KMutexRelease(&space->data.mutex)); - - uintptr_t tableBase = virtualAddressStart & 0x0000FFFFFFFFF000; - uintptr_t start = resumePosition ? *resumePosition : 0; - - // TODO Freeing newly empty page tables. - // - What do we need to invalidate when we do this? - - for (uintptr_t i = start; i < pageCount; i++) { - uintptr_t virtualAddress = (i << K_PAGE_BITS) + tableBase; - - if ((PAGE_TABLE_L4[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3)] & 1) == 0) { - i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 3)); - i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 3)); - continue; - } - - if ((PAGE_TABLE_L3[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2)] & 1) == 0) { - i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 2)); - i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 2)); - continue; - } - - if ((PAGE_TABLE_L2[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)] & 1) == 0) { - i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 1)); - i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 1)); - continue; - } - - uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); - - uintptr_t translation = PAGE_TABLE_L1[indexL1]; - - if (!(translation & 1)) { - // The page wasn't mapped. - continue; - } - - bool copy = translation & (1 << 9); - - if (copy && (flags & MM_UNMAP_PAGES_BALANCE_FILE) && (~flags & MM_UNMAP_PAGES_FREE_COPIED)) { - // Ignore copied pages when balancing file mappings. - continue; - } - - if ((~translation & (1 << 5)) || (~translation & (1 << 6))) { - // See MMArchMapPage for a discussion of why these bits must be set. - KernelPanic("MMArchUnmapPages - Page found without accessed or dirty bit set (virtualAddress: %x, translation: %x).\n", - virtualAddress, translation); - } - - PAGE_TABLE_L1[indexL1] = 0; - - uintptr_t physicalAddress = translation & 0x0000FFFFFFFFF000; - - if ((flags & MM_UNMAP_PAGES_FREE) || ((flags & MM_UNMAP_PAGES_FREE_COPIED) && copy)) { - MMPhysicalFree(physicalAddress, true); - } else if (flags & MM_UNMAP_PAGES_BALANCE_FILE) { - // It's safe to do this before page invalidation, - // because the page fault handler is synchronised with the same mutexes acquired above. - - if (MMUnmapFilePage(physicalAddress >> K_PAGE_BITS)) { - if (resumePosition) { - if (!unmapMaximum--) { - *resumePosition = i; - break; - } - } - } - } - } - - MMArchInvalidatePages(virtualAddressStart, pageCount); -} - InterruptContext *ArchInitialiseThread(uintptr_t kernelStack, uintptr_t kernelStackSize, Thread *thread, uintptr_t startAddress, uintptr_t argument1, uintptr_t argument2, bool userland, uintptr_t stack, uintptr_t userStackSize) { @@ -605,20 +324,20 @@ void MMArchFreeVAS(MMSpace *space) { for (uintptr_t i = 0; i < 256; i++) { if (!PAGE_TABLE_L4[i]) continue; - for (uintptr_t j = i * 512; j < (i + 1) * 512; j++) { + for (uintptr_t j = i * ENTRIES_PER_PAGE_TABLE; j < (i + 1) * ENTRIES_PER_PAGE_TABLE; j++) { if (!PAGE_TABLE_L3[j]) continue; - for (uintptr_t k = j * 512; k < (j + 1) * 512; k++) { + for (uintptr_t k = j * ENTRIES_PER_PAGE_TABLE; k < (j + 1) * ENTRIES_PER_PAGE_TABLE; k++) { if (!PAGE_TABLE_L2[k]) continue; - MMPhysicalFree(PAGE_TABLE_L2[k] & (~0xFFF)); + MMPhysicalFree(PAGE_TABLE_L2[k] & ~(K_PAGE_SIZE - 1)); space->data.pageTablesActive--; } - MMPhysicalFree(PAGE_TABLE_L3[j] & (~0xFFF)); + MMPhysicalFree(PAGE_TABLE_L3[j] & ~(K_PAGE_SIZE - 1)); space->data.pageTablesActive--; } - MMPhysicalFree(PAGE_TABLE_L4[i] & (~0xFFF)); + MMPhysicalFree(PAGE_TABLE_L4[i] & ~(K_PAGE_SIZE - 1)); space->data.pageTablesActive--; } @@ -644,334 +363,6 @@ void MMArchFinalizeVAS(MMSpace *space) { MMDecommit(K_PAGE_SIZE, true); } -uint64_t ArchGetTimeFromPITMs() { - // TODO This isn't working on real hardware, but ArchDelay1Ms is? - - // NOTE This will only work if called at least once every 50 ms. - // (The PIT only stores a 16-bit counter, which is depleted every 50 ms.) - - static bool started = false; - static uint64_t cumulative = 0, last = 0; - - if (!started) { - ProcessorOut8(IO_PIT_COMMAND, 0x30); - ProcessorOut8(IO_PIT_DATA, 0xFF); - ProcessorOut8(IO_PIT_DATA, 0xFF); - started = true; - last = 0xFFFF; - return 0; - } else { - ProcessorOut8(IO_PIT_COMMAND, 0x00); - uint16_t x = ProcessorIn8(IO_PIT_DATA); - x |= (ProcessorIn8(IO_PIT_DATA)) << 8; - cumulative += last - x; - if (x > last) cumulative += 0x10000; - last = x; - return cumulative * 1000 / 1193182; - } -} - -void ArchDelay1Ms() { - ProcessorOut8(IO_PIT_COMMAND, 0x30); - ProcessorOut8(IO_PIT_DATA, 0xA9); - ProcessorOut8(IO_PIT_DATA, 0x04); - - while (true) { - ProcessorOut8(IO_PIT_COMMAND, 0xE2); - - if (ProcessorIn8(IO_PIT_DATA) & (1 << 7)) { - break; - } - } -} - -bool SetupInterruptRedirectionEntry(uintptr_t _line) { - KSpinlockAssertLocked(&scheduler.lock); - - static uint32_t alreadySetup = 0; - - if (alreadySetup & (1 << _line)) { - return true; - } - - // Work out which interrupt the IoApic will sent to the processor. - // TODO Use the upper 4 bits for IRQ priority. - - uintptr_t line = _line; - uintptr_t thisProcessorIRQ = line + IRQ_BASE; - - bool activeLow = false; - bool levelTriggered = true; - - // If there was an interrupt override entry in the MADT table, - // then we'll have to use that number instead. - - for (uintptr_t i = 0; i < acpi.interruptOverrideCount; i++) { - ACPIInterruptOverride *interruptOverride = acpi.interruptOverrides + i; - - if (interruptOverride->sourceIRQ == line) { - line = interruptOverride->gsiNumber; - activeLow = interruptOverride->activeLow; - levelTriggered = interruptOverride->levelTriggered; - break; - } - } - - KernelLog(LOG_INFO, "Arch", "IRQ flags", "SetupInterruptRedirectionEntry - IRQ %d is active %z, %z triggered.\n", - line, activeLow ? "low" : "high", levelTriggered ? "level" : "edge"); - - ACPIIoApic *ioApic; - bool foundIoApic = false; - - // Look for the IoApic to which this interrupt is sent. - - for (uintptr_t i = 0; i < acpi.ioapicCount; i++) { - ioApic = acpi.ioApics + i; - if (line >= ioApic->gsiBase && line < (ioApic->gsiBase + (0xFF & (ACPIIoApicReadRegister(ioApic, 1) >> 16)))) { - foundIoApic = true; - line -= ioApic->gsiBase; - break; - } - } - - // We couldn't find the IoApic that handles this interrupt. - - if (!foundIoApic) { - KernelLog(LOG_ERROR, "Arch", "no IOAPIC", "SetupInterruptRedirectionEntry - Could not find an IOAPIC handling interrupt line %d.\n", line); - return false; - } - - // A normal priority interrupt. - - uintptr_t redirectionTableIndex = line * 2 + 0x10; - uint32_t redirectionEntry = thisProcessorIRQ; - if (activeLow) redirectionEntry |= (1 << 13); - if (levelTriggered) redirectionEntry |= (1 << 15); - - // Send the interrupt to the processor that registered the interrupt. - - ACPIIoApicWriteRegister(ioApic, redirectionTableIndex, 1 << 16); // Mask the interrupt while we modify the entry. - ACPIIoApicWriteRegister(ioApic, redirectionTableIndex + 1, GetLocalStorage()->archCPU->apicID << 24); - ACPIIoApicWriteRegister(ioApic, redirectionTableIndex, redirectionEntry); - - alreadySetup |= 1 << _line; - return true; -} - -void KUnregisterMSI(uintptr_t tag) { - KSpinlockAcquire(&irqHandlersLock); - EsDefer(KSpinlockRelease(&irqHandlersLock)); - msiHandlers[tag].callback = nullptr; -} - -KMSIInformation KRegisterMSI(KIRQHandler handler, void *context, const char *cOwnerName) { - KSpinlockAcquire(&irqHandlersLock); - EsDefer(KSpinlockRelease(&irqHandlersLock)); - - for (uintptr_t i = 0; i < INTERRUPT_VECTOR_MSI_COUNT; i++) { - if (msiHandlers[i].callback) continue; - msiHandlers[i] = { handler, context }; - - // TODO Selecting the best target processor. - // Currently this sends everything to processor 0. - - KernelLog(LOG_INFO, "Arch", "register MSI", "Register MSI with vector %X for '%z'.\n", - INTERRUPT_VECTOR_MSI_START + i, cOwnerName); - - return { - .address = 0xFEE00000, - .data = INTERRUPT_VECTOR_MSI_START + i, - .tag = i, - }; - } - - return {}; -} - -bool KRegisterIRQ(intptr_t line, KIRQHandler handler, void *context, const char *cOwnerName, KPCIDevice *pciDevice) { - KSpinlockAcquire(&scheduler.lock); - EsDefer(KSpinlockRelease(&scheduler.lock)); - - if (line == -1 && !pciDevice) { - KernelPanic("KRegisterIRQ - Interrupt line is %d, and pciDevice is %x.\n", line, pciDevice); - } - - // Save the handler callback and context. - - if (line > 0x20 || line < -1) KernelPanic("KRegisterIRQ - Unexpected IRQ %d\n", line); - bool found = false; - - KSpinlockAcquire(&irqHandlersLock); - - for (uintptr_t i = 0; i < sizeof(irqHandlers) / sizeof(irqHandlers[0]); i++) { - if (!irqHandlers[i].callback) { - found = true; - irqHandlers[i].callback = handler; - irqHandlers[i].context = context; - irqHandlers[i].line = line; - irqHandlers[i].pciDevice = pciDevice; - irqHandlers[i].cOwnerName = cOwnerName; - break; - } - } - - KSpinlockRelease(&irqHandlersLock); - - if (!found) { - KernelLog(LOG_ERROR, "Arch", "too many IRQ handlers", "The limit of IRQ handlers was reached (%d), and the handler for '%z' was not registered.\n", - sizeof(irqHandlers) / sizeof(irqHandlers[0]), cOwnerName); - return false; - } - - KernelLog(LOG_INFO, "Arch", "register IRQ", "KRegisterIRQ - Registered IRQ %d to '%z'.\n", line, cOwnerName); - - if (line != -1) { - if (!SetupInterruptRedirectionEntry(line)) { - return false; - } - } else { - SetupInterruptRedirectionEntry(9); - SetupInterruptRedirectionEntry(10); - SetupInterruptRedirectionEntry(11); - } - - return true; -} - -size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi, int processorID) { - // It's possible that another CPU is trying to send an IPI at the same time we want to send the panic IPI. - // TODO What should we do in this case? - if (interrupt != KERNEL_PANIC_IPI) KSpinlockAssertLocked(&ipiLock); - - // Note: We send IPIs at a special priority that ProcessorDisableInterrupts doesn't mask. - - size_t ignored = 0; - - for (uintptr_t i = 0; i < acpi.processorCount; i++) { - ArchCPU *processor = acpi.processors + i; - - if (processorID != -1) { - if (processorID != processor->kernelProcessorID) { - ignored++; - continue; - } - } else { - if (processor == GetLocalStorage()->archCPU || !processor->local || !processor->local->schedulerReady) { - ignored++; - continue; - } - } - - uint32_t destination = acpi.processors[i].apicID << 24; - uint32_t command = interrupt | (1 << 14) | (nmi ? 0x400 : 0); - LapicWriteRegister(0x310 >> 2, destination); - LapicWriteRegister(0x300 >> 2, command); - - // Wait for the interrupt to be sent. - while (LapicReadRegister(0x300 >> 2) & (1 << 12)); - } - - return ignored; -} - -void ProcessorSendYieldIPI(Thread *thread) { - thread->receivedYieldIPI = false; - KSpinlockAcquire(&ipiLock); - ProcessorSendIPI(YIELD_IPI, false); - KSpinlockRelease(&ipiLock); - while (!thread->receivedYieldIPI); // Spin until the thread gets the IPI. -} - -void ArchNextTimer(size_t ms) { - while (!scheduler.started); // Wait until the scheduler is ready. - GetLocalStorage()->schedulerReady = true; // Make sure this CPU can be scheduled. - LapicNextTimer(ms); // Set the next timer. -} - -NewProcessorStorage AllocateNewProcessorStorage(ArchCPU *archCPU) { - NewProcessorStorage storage = {}; - storage.local = (CPULocalStorage *) EsHeapAllocate(sizeof(CPULocalStorage), true, K_FIXED); - storage.gdt = (uint32_t *) MMMapPhysical(kernelMMSpace, MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_COMMIT_NOW), K_PAGE_SIZE, ES_FLAGS_DEFAULT); - storage.local->archCPU = archCPU; - archCPU->local = storage.local; - scheduler.CreateProcessorThreads(storage.local); - archCPU->kernelProcessorID = storage.local->processorID; - return storage; -} - -void SetupProcessor2(NewProcessorStorage *storage) { - // Setup the local interrupts for the current processor. - - for (uintptr_t i = 0; i < acpi.lapicNMICount; i++) { - if (acpi.lapicNMIs[i].processor == 0xFF - || acpi.lapicNMIs[i].processor == storage->local->archCPU->processorID) { - uint32_t registerIndex = (0x350 + (acpi.lapicNMIs[i].lintIndex << 4)) >> 2; - uint32_t value = 2 | (1 << 10); // NMI exception interrupt vector. - if (acpi.lapicNMIs[i].activeLow) value |= 1 << 13; - if (acpi.lapicNMIs[i].levelTriggered) value |= 1 << 15; - LapicWriteRegister(registerIndex, value); - } - } - - LapicWriteRegister(0x350 >> 2, LapicReadRegister(0x350 >> 2) & ~(1 << 16)); - LapicWriteRegister(0x360 >> 2, LapicReadRegister(0x360 >> 2) & ~(1 << 16)); - LapicWriteRegister(0x080 >> 2, 0); - if (LapicReadRegister(0x30 >> 2) & 0x80000000) LapicWriteRegister(0x410 >> 2, 0); - LapicEndOfInterrupt(); - - // Configure the LAPIC's timer. - - LapicWriteRegister(0x3E0 >> 2, 2); // Divisor = 16 - - // Create the processor's local storage. - - ProcessorSetLocalStorage(storage->local); - - // Setup a GDT and TSS for the processor. - - uint32_t *gdt = storage->gdt; - void *bootstrapGDT = (void *) (((uint64_t *) ((uint16_t *) processorGDTR + 1))[0]); - EsMemoryCopy(gdt, bootstrapGDT, 2048); - uint32_t *tss = (uint32_t *) ((uint8_t *) storage->gdt + 2048); - storage->local->archCPU->kernelStack = (void **) (tss + 1); - ProcessorInstallTSS(gdt, tss); -} - -const char *exceptionInformation[] = { - "0x00: Divide Error (Fault)", - "0x01: Debug Exception (Fault/Trap)", - "0x02: Non-Maskable External Interrupt (Interrupt)", - "0x03: Breakpoint (Trap)", - "0x04: Overflow (Trap)", - "0x05: BOUND Range Exceeded (Fault)", - "0x06: Invalid Opcode (Fault)", - "0x07: x87 Coprocessor Unavailable (Fault)", - "0x08: Double Fault (Abort)", - "0x09: x87 Coprocessor Segment Overrun (Fault)", - "0x0A: Invalid TSS (Fault)", - "0x0B: Segment Not Present (Fault)", - "0x0C: Stack Protection (Fault)", - "0x0D: General Protection (Fault)", - "0x0E: Page Fault (Fault)", - "0x0F: Reserved/Unknown", - "0x10: x87 FPU Floating-Point Error (Fault)", - "0x11: Alignment Check (Fault)", - "0x12: Machine Check (Abort)", - "0x13: SIMD Floating-Point Exception (Fault)", - "0x14: Virtualization Exception (Fault)", - "0x15: Reserved/Unknown", - "0x16: Reserved/Unknown", - "0x17: Reserved/Unknown", - "0x18: Reserved/Unknown", - "0x19: Reserved/Unknown", - "0x1A: Reserved/Unknown", - "0x1B: Reserved/Unknown", - "0x1C: Reserved/Unknown", - "0x1D: Reserved/Unknown", - "0x1E: Reserved/Unknown", - "0x1F: Reserved/Unknown", -}; - void ContextSanityCheck(InterruptContext *context) { if (!context || context->cs > 0x100 || context->ds > 0x100 || context->ss > 0x100 || (context->rip >= 0x1000000000000 && context->rip < 0xFFFF000000000000) @@ -1170,12 +561,12 @@ extern "C" void InterruptHandler(InterruptContext *context) { handler.callback(interrupt - INTERRUPT_VECTOR_MSI_START, handler.context); } - LapicEndOfInterrupt(); - if (local->irqSwitchThread && scheduler.started && local->schedulerReady) { - scheduler.Yield(context); + scheduler.Yield(context); // LapicEndOfInterrupt is called in PostContextSwitch. KernelPanic("InterruptHandler - Returned from Scheduler::Yield.\n"); } + + LapicEndOfInterrupt(); } else if (local) { // IRQ. @@ -1230,12 +621,12 @@ extern "C" void InterruptHandler(InterruptContext *context) { GetLocalStorage()->inIRQ = false; } - LapicEndOfInterrupt(); - if (local->irqSwitchThread && scheduler.started && local->schedulerReady) { - scheduler.Yield(context); + scheduler.Yield(context); // LapicEndOfInterrupt is called in PostContextSwitch. KernelPanic("InterruptHandler - Returned from Scheduler::Yield.\n"); } + + LapicEndOfInterrupt(); } // Sanity check. @@ -1246,87 +637,12 @@ extern "C" void InterruptHandler(InterruptContext *context) { } } -extern "C" bool PostContextSwitch(InterruptContext *context, MMSpace *oldAddressSpace) { - CPULocalStorage *local = GetLocalStorage(); - Thread *currentThread = GetCurrentThread(); - - void *kernelStack = (void *) currentThread->kernelStack; - *local->archCPU->kernelStack = kernelStack; - - bool newThread = currentThread->cpuTimeSlices == 1; - - KernelLog(LOG_VERBOSE, "Arch", "context switch", "Context switch to thread %x at %x\n", currentThread, context->rip); - - if (newThread) { - ContextSanityCheck(context); - KernelLog(LOG_VERBOSE, "Arch", "executing new thread", "Executing new thread %x at %x\n", currentThread, context->rip); - } - - LapicEndOfInterrupt(); - ContextSanityCheck(context); - - if (ProcessorAreInterruptsEnabled()) { - KernelPanic("PostContextSwitch - Interrupts were enabled. (1)\n"); - } - - currentThread->lastKnownExecutionAddress = context->rip; - - if (scheduler.lock.interruptsEnabled) { - KernelPanic("PostContextSwitch - Interrupts were enabled. (3)\n"); - } - - ProcessorSetThreadStorage(currentThread->tlsAddress); - - // We can only free the scheduler's spinlock when we are no longer using the stack - // from the previous thread. See DoContextSwitch in x86_64.s. - // (Another CPU can KillThread this once it's back in activeThreads.) - KSpinlockRelease(&scheduler.lock, true); - - if (ProcessorAreInterruptsEnabled()) { - KernelPanic("PostContextSwitch - Interrupts were enabled. (2)\n"); - } - - MMSpaceCloseReference(oldAddressSpace); - - return newThread; -} - extern "C" uintptr_t Syscall(uintptr_t argument0, uintptr_t argument1, uintptr_t argument2, uintptr_t returnAddress, uintptr_t argument3, uintptr_t argument4, uintptr_t *userStackPointer) { (void) returnAddress; return DoSyscall((EsSyscallType) argument0, argument1, argument2, argument3, argument4, false, nullptr, userStackPointer); } -uintptr_t GetBootloaderInformationOffset() { - return bootloaderInformationOffset; -} - -// Spinlock since some drivers need to access it in IRQs (e.g. ACPICA). -static KSpinlock pciConfigSpinlock; - -uint32_t KPCIReadConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, int size) { - KSpinlockAcquire(&pciConfigSpinlock); - EsDefer(KSpinlockRelease(&pciConfigSpinlock)); - if (offset & 3) KernelPanic("KPCIReadConfig - offset is not 4-byte aligned."); - ProcessorOut32(IO_PCI_CONFIG, (uint32_t) (0x80000000 | (bus << 16) | (device << 11) | (function << 8) | offset)); - if (size == 8) return ProcessorIn8(IO_PCI_DATA); - if (size == 16) return ProcessorIn16(IO_PCI_DATA); - if (size == 32) return ProcessorIn32(IO_PCI_DATA); - KernelPanic("PCIController::ReadConfig - Invalid size %d.\n", size); - return 0; -} - -void KPCIWriteConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value, int size) { - KSpinlockAcquire(&pciConfigSpinlock); - EsDefer(KSpinlockRelease(&pciConfigSpinlock)); - if (offset & 3) KernelPanic("KPCIWriteConfig - offset is not 4-byte aligned."); - ProcessorOut32(IO_PCI_CONFIG, (uint32_t) (0x80000000 | (bus << 16) | (device << 11) | (function << 8) | offset)); - if (size == 8) ProcessorOut8(IO_PCI_DATA, value); - else if (size == 16) ProcessorOut16(IO_PCI_DATA, value); - else if (size == 32) ProcessorOut32(IO_PCI_DATA, value); - else KernelPanic("PCIController::WriteConfig - Invalid size %d.\n", size); -} - EsError ArchApplyRelocation(uintptr_t type, uint8_t *buffer, uintptr_t offset, uintptr_t result) { if (type == 0) {} else if (type == 10 /* R_X86_64_32 */) *((uint32_t *) (buffer + offset)) = result; @@ -1339,49 +655,6 @@ EsError ArchApplyRelocation(uintptr_t type, uint8_t *buffer, uintptr_t offset, u return ES_SUCCESS; } -uintptr_t MMArchEarlyAllocatePage() { - uintptr_t i = physicalMemoryRegionsIndex; - - while (!physicalMemoryRegions[i].pageCount) { - i++; - - if (i == physicalMemoryRegionsCount) { - KernelPanic("MMArchEarlyAllocatePage - Expected more pages in physical regions.\n"); - } - } - - PhysicalMemoryRegion *region = physicalMemoryRegions + i; - uintptr_t returnValue = region->baseAddress; - - region->baseAddress += K_PAGE_SIZE; - region->pageCount--; - physicalMemoryRegionsPagesCount--; - physicalMemoryRegionsIndex = i; - - return returnValue; -} - -uint64_t MMArchPopulatePageFrameDatabase() { - uint64_t commitLimit = 0; - - for (uintptr_t i = 0; i < physicalMemoryRegionsCount; i++) { - uintptr_t base = physicalMemoryRegions[i].baseAddress >> K_PAGE_BITS; - uintptr_t count = physicalMemoryRegions[i].pageCount; - commitLimit += count; - - for (uintptr_t j = 0; j < count; j++) { - MMPhysicalInsertFreePagesNext(base + j); - } - } - - physicalMemoryRegionsPagesCount = 0; - return commitLimit; -} - -uintptr_t MMArchGetPhysicalMemoryHighest() { - return physicalMemoryHighest; -} - void ArchStartupApplicationProcessors() { // TODO How do we know that this address is usable? #define AP_TRAMPOLINE 0x10000 @@ -1481,103 +754,6 @@ void ArchStartupApplicationProcessors() { MMArchUnmapPages(kernelMMSpace, AP_TRAMPOLINE, 2, ES_FLAGS_DEFAULT); } -uint64_t ArchGetTimeMs() { - // Update the time stamp counter synchronization value. - timeStampCounterSynchronizationValue = ((timeStampCounterSynchronizationValue & 0x8000000000000000) - ^ 0x8000000000000000) | ProcessorReadTimeStamp(); - - if (acpi.hpetBaseAddress && acpi.hpetPeriod) { - __int128 fsToMs = 1000000000000; - __int128 reading = acpi.hpetBaseAddress[30]; - return (uint64_t) (reading * (__int128) acpi.hpetPeriod / fsToMs); - } else { - return ArchGetTimeFromPITMs(); - } -} - -uintptr_t ArchFindRootSystemDescriptorPointer() { - uint64_t uefiRSDP = *((uint64_t *) (LOW_MEMORY_MAP_START + GetBootloaderInformationOffset() + 0x7FE8)); - - if (uefiRSDP) { - return uefiRSDP; - } - - PhysicalMemoryRegion searchRegions[2]; - - searchRegions[0].baseAddress = (uintptr_t) (((uint16_t *) LOW_MEMORY_MAP_START)[0x40E] << 4) + LOW_MEMORY_MAP_START; - searchRegions[0].pageCount = 0x400; - searchRegions[1].baseAddress = (uintptr_t) 0xE0000 + LOW_MEMORY_MAP_START; - searchRegions[1].pageCount = 0x20000; - - for (uintptr_t i = 0; i < 2; i++) { - for (uintptr_t address = searchRegions[i].baseAddress; - address < searchRegions[i].baseAddress + searchRegions[i].pageCount; - address += 16) { - RootSystemDescriptorPointer *rsdp = (RootSystemDescriptorPointer *) address; - - if (rsdp->signature != SIGNATURE_RSDP) { - continue; - } - - if (rsdp->revision == 0) { - if (EsMemorySumBytes((uint8_t *) rsdp, 20)) { - continue; - } - - return (uintptr_t) rsdp - LOW_MEMORY_MAP_START; - } else if (rsdp->revision == 2) { - if (EsMemorySumBytes((uint8_t *) rsdp, sizeof(RootSystemDescriptorPointer))) { - continue; - } - - return (uintptr_t) rsdp - LOW_MEMORY_MAP_START; - } - } - } - - return 0; -} - -void ArchInitialise() { - ACPIParseTables(); - - uint8_t bootstrapLapicID = (LapicReadRegister(0x20 >> 2) >> 24); - - ArchCPU *currentCPU = nullptr; - - for (uintptr_t i = 0; i < acpi.processorCount; i++) { - if (acpi.processors[i].apicID == bootstrapLapicID) { - // That's us! - currentCPU = acpi.processors + i; - currentCPU->bootProcessor = true; - break; - } - } - - if (!currentCPU) { - KernelPanic("ACPIInitialise - Could not find the bootstrap processor\n"); - } - - // Calibrate the LAPIC's timer and processor's timestamp counter. - ProcessorDisableInterrupts(); - uint64_t start = ProcessorReadTimeStamp(); - LapicWriteRegister(0x380 >> 2, (uint32_t) -1); - for (int i = 0; i < 8; i++) ArchDelay1Ms(); // Average over 8ms - acpi.lapicTicksPerMs = ((uint32_t) -1 - LapicReadRegister(0x390 >> 2)) >> 4; - EsRandomAddEntropy(LapicReadRegister(0x390 >> 2)); - uint64_t end = ProcessorReadTimeStamp(); - timeStampTicksPerMs = (end - start) >> 3; - ProcessorEnableInterrupts(); - // EsPrint("timeStampTicksPerMs = %d\n", timeStampTicksPerMs); - - // Finish processor initialisation. - // This sets up interrupts, the timer, CPULocalStorage, the GDT and TSS, - // and registers the processor with the scheduler. - - NewProcessorStorage storage = AllocateNewProcessorStorage(currentCPU); - SetupProcessor2(&storage); -} - #include #endif diff --git a/arch/x86_64/kernel.s b/arch/x86_64/kernel.s index bf7703a..a2bcdb5 100644 --- a/arch/x86_64/kernel.s +++ b/arch/x86_64/kernel.s @@ -1,5 +1,61 @@ [bits 64] +[global ArchSwitchContext] +[global GetCurrentThread] +[global GetLocalStorage] +[global MMArchSafeCopy] +[global ProcessorAPStartup] +[global ProcessorAreInterruptsEnabled] +[global ProcessorDebugOutputByte] +[global ProcessorDisableInterrupts] +[global ProcessorEnableInterrupts] +[global ProcessorFakeTimerInterrupt] +[global ProcessorFlushCodeCache] +[global ProcessorGetRBP] +[global ProcessorGetRSP] +[global ProcessorHalt] +[global ProcessorIn16] +[global ProcessorIn32] +[global ProcessorIn8] +[global ProcessorInstallTSS] +[global ProcessorInvalidateAllPages] +[global ProcessorInvalidatePage] +[global ProcessorOut16] +[global ProcessorOut32] +[global ProcessorOut8] +[global ProcessorReadCR3] +[global ProcessorReadMXCSR] +[global ProcessorReadTimeStamp] +[global ProcessorReset] +[global ProcessorSetAddressSpace] +[global ProcessorSetLocalStorage] +[global ProcessorSetThreadStorage] +[global _KThreadTerminate] +[global _start] +[global gdt_data] +[global pagingNXESupport] +[global pagingPCIDSupport] +[global pagingSMEPSupport] +[global pagingTCESupport] +[global processorGDTR] +[global simdSSE3Support] +[global simdSSSE3Support] +[global timeStampCounterSynchronizationValue] + +[extern ArchNextTimer] +[extern InterruptHandler] +[extern KThreadTerminate] +[extern KernelMain] +[extern PostContextSwitch] +[extern SetupProcessor2] +[extern Syscall] +[extern installationID] +[extern PCSetupCOM1] +[extern PCDisablePIC] +[extern PCProcessMemoryMap] +[extern bootloaderID] +[extern bootloaderInformationOffset] + [section .bss] align 16 @@ -23,62 +79,27 @@ idt: cpu_local_storage_index: dq 0 -[global physicalMemoryRegions] -physicalMemoryRegions: - dq 0xFFFFFE0000060000 -[global physicalMemoryRegionsCount] -physicalMemoryRegionsCount: - dq 0 -[global physicalMemoryRegionsPagesCount] -physicalMemoryRegionsPagesCount: - dq 0 -[global physicalMemoryOriginalPagesCount] -physicalMemoryOriginalPagesCount: - dq 0 -[global physicalMemoryRegionsIndex] -physicalMemoryRegionsIndex: - dq 0 -[global physicalMemoryHighest] -physicalMemoryHighest: - dq 0 - -[global pagingNXESupport] pagingNXESupport: dd 1 -[global pagingPCIDSupport] pagingPCIDSupport: dd 1 -[global pagingSMEPSupport] pagingSMEPSupport: dd 1 -[global pagingTCESupport] pagingTCESupport: dd 1 -[global simdSSE3Support] simdSSE3Support: dd 1 -[global simdSSSE3Support] simdSSSE3Support: dd 1 -[global bootloaderID] -bootloaderID: - dd 0 -[global bootloaderInformationOffset] -bootloaderInformationOffset: - dq 0 - align 16 -[global processorGDTR] processorGDTR: dq 0 dq 0 [section .text] -[global _start] _start: - cli mov rax,0x63 mov fs,ax mov gs,ax @@ -87,20 +108,20 @@ _start: mov rax,bootloaderID mov [rax],rsi + ; The MBR bootloader does not know the address of the RSDP. cmp rdi,0 jne .standard_acpi mov [0x7FE8],rdi .standard_acpi: - ; Install a stack - mov rsp,stack + stack_size - ; Save the bootloader information offset. mov rax,bootloaderInformationOffset mov [rax],rdi + ; Install a stack + mov rsp,stack + stack_size + ; Load the installation ID. -[extern installationID] mov rbx,installationID mov rax,[rdi + 0x7FF0] mov [rbx],rax @@ -113,57 +134,9 @@ _start: mov rax,cr3 mov cr3,rax -SetupCOM1: - ; Setup the serial COM1 port for debug output. -%ifdef COM_OUTPUT - mov dx,0x3F8 + 1 - mov al,0x00 - out dx,al - mov dx,0x3F8 + 3 - mov al,0x80 - out dx,al - mov dx,0x3F8 + 0 - mov al,0x03 - out dx,al - mov dx,0x3F8 + 1 - mov al,0x00 - out dx,al - mov dx,0x3F8 + 3 - mov al,0x03 - out dx,al - mov dx,0x3F8 + 2 - mov al,0xC7 - out dx,al - mov dx,0x3F8 + 4 - mov al,0x0B - out dx,al -%endif - -InstallIDT: - ; Remap the ISRs sent by the PIC to 0x20 - 0x2F - ; Even though we'll mask the PIC to use the APIC, - ; we have to do this so that the spurious interrupts - ; are set to a sane vector range. - mov al,0x11 - out 0x20,al - mov al,0x11 - out 0xA0,al - mov al,0x20 - out 0x21,al - mov al,0x28 - out 0xA1,al - mov al,0x04 - out 0x21,al - mov al,0x02 - out 0xA1,al - mov al,0x01 - out 0x21,al - mov al,0x01 - out 0xA1,al - mov al,0x00 - out 0x21,al - mov al,0x00 - out 0xA1,al + call PCSetupCOM1 + call PCDisablePIC + call PCProcessMemoryMap ; Install the interrupt handlers %macro INSTALL_INTERRUPT_HANDLER 1 @@ -181,78 +154,21 @@ InstallIDT: mov rcx,processorGDTR sgdt [rcx] -MemoryCalculations: - ; Work out basic information about the physical memory map we got from the bootloader - mov rax,bootloaderInformationOffset - mov rax,[rax] - mov rbx,physicalMemoryRegions - add [rbx],rax - mov rdi,0xFFFFFE0000060000 - 0x10 - add rdi,rax - mov rsi,0xFFFFFE0000060000 - add rsi,rax - xor rax,rax - xor r8,r8 - .loop: - add rdi,0x10 - mov r9,[rdi + 8] - shl r9,12 - add r9,[rdi] - cmp r9,r8 - jb .lower - mov r8,r9 - .lower: - add rax,[rdi + 8] - cmp qword [rdi],0 - jne .loop - mov rbx,[rdi + 8] - sub rax,rbx - sub rdi,rsi - shr rdi,4 - mov rsi,physicalMemoryRegionsCount - mov [rsi],rdi - mov rsi,physicalMemoryRegionsPagesCount - mov [rsi],rax - mov rsi,physicalMemoryOriginalPagesCount - mov [rsi],rbx - mov rsi,physicalMemoryHighest - mov [rsi],r8 - -DisablePIC: - ; Disable the PIC by masking all its interrupts, as we're going to use the APIC instead. - ; For some reason, it'll still generate spurious interrupts, so we'll have to ignore those. - mov al,0xFF - out 0xA1,al - out 0x21,al - -StartKernel: ; First stage of processor initilisation call SetupProcessor1 - ; Print a divider line. - mov rdi,'-' - mov rcx,10 - .line: call ProcessorDebugOutputByte - loop .line - mov rdi,10 - call ProcessorDebugOutputByte - mov rdi,13 - call ProcessorDebugOutputByte - ; Call the KernelMain function and rsp,~0xF - extern KernelMain call KernelMain ProcessorReady: ; Set the timer and become this CPU's idle thread. mov rdi,1 - [extern ArchNextTimer] call ArchNextTimer jmp ProcessorIdle SetupProcessor1: -EnableCPUFeatures: + .enable_cpu_features: ; Enable no-execute support, if available mov eax,0x80000001 cpuid @@ -391,7 +307,7 @@ EnableCPUFeatures: or eax,0x00010000 wrmsr -SetupCPULocalStorage: + .setup_cpu_local_storage: mov ecx,0xC0000101 mov rax,cpu_local_storage mov rdx,cpu_local_storage @@ -401,13 +317,13 @@ SetupCPULocalStorage: add qword [rdi],32 ; Space for 4 8-byte values at gs:0 - gs:31 wrmsr -LoadIDTR: + .load_idtr: ; Load the IDTR mov rax,idt lidt [rax] sti -EnableAPIC: + .enable_apic: ; Enable the APIC! ; Since we're on AMD64, we know that the APIC will be present. mov ecx,0x1B @@ -453,7 +369,6 @@ SyscallEntry: ; Arguments in RDI, RSI, RDX, R8, R9. (RCX contains return address). ; Return value in RAX. - [extern Syscall] mov rbx,rsp and rsp,~0xF call Syscall @@ -476,19 +391,16 @@ SyscallEntry: db 0x48 sysret -[global ProcessorFakeTimerInterrupt] ProcessorFakeTimerInterrupt: int 0x40 ret -[global ProcessorDisableInterrupts] ProcessorDisableInterrupts: mov rax,14 ; Still allow important IPIs to go through. mov cr8,rax sti ; TODO Where is this necessary? Is is a performance issue? ret -[global ProcessorEnableInterrupts] ProcessorEnableInterrupts: ; WARNING: Changing this mechanism also requires update in x86_64.cpp, when deciding if we should re-enable interrupts on exception. mov rax,0 @@ -496,7 +408,6 @@ ProcessorEnableInterrupts: sti ; TODO Where is this necessary? Is is a performance issue? ret -[global ProcessorAreInterruptsEnabled] ProcessorAreInterruptsEnabled: pushf pop rax @@ -515,60 +426,51 @@ ProcessorAreInterruptsEnabled: ; shr rax,9 ret -[global ProcessorHalt] ProcessorHalt: cli hlt jmp ProcessorHalt -[global ProcessorOut8] ProcessorOut8: mov rdx,rdi mov rax,rsi out dx,al ret -[global ProcessorIn8] ProcessorIn8: mov rdx,rdi xor rax,rax in al,dx ret -[global ProcessorOut16] ProcessorOut16: mov rdx,rdi mov rax,rsi out dx,ax ret -[global ProcessorIn16] ProcessorIn16: mov rdx,rdi xor rax,rax in ax,dx ret -[global ProcessorOut32] ProcessorOut32: mov rdx,rdi mov rax,rsi out dx,eax ret -[global ProcessorIn32] ProcessorIn32: mov rdx,rdi xor rax,rax in eax,dx ret -[global ProcessorInvalidatePage] ProcessorInvalidatePage: invlpg [rdi] ret -[global ProcessorInvalidateAllPages] ProcessorInvalidateAllPages: ; Toggle CR4.PGE to invalidate all TLB entries, including global entries. mov rax,cr4 @@ -583,22 +485,18 @@ ProcessorIdle: hlt jmp ProcessorIdle -[global GetLocalStorage] GetLocalStorage: mov rax,[gs:0] ret -[global GetCurrentThread] GetCurrentThread: mov rax,[gs:16] ret -[global ProcessorSetLocalStorage] ProcessorSetLocalStorage: mov [gs:0],rdi ret -[global ProcessorSetThreadStorage] ProcessorSetThreadStorage: push rdx push rcx @@ -718,7 +616,6 @@ ASMInterruptHandler: mov rdi,rsp mov rbx,rsp and rsp,~0xF - extern InterruptHandler call InterruptHandler mov rsp,rbx xor rax,rax @@ -767,7 +664,6 @@ ReturnFromInterruptHandler: add rsp,16 iretq -[global ProcessorSetAddressSpace] ProcessorSetAddressSpace: mov rdi,[rdi] mov rax,cr3 @@ -777,18 +673,14 @@ ProcessorSetAddressSpace: .cont: ret -[global ProcessorGetRSP] ProcessorGetRSP: mov rax,rsp ret -[global ProcessorGetRBP] ProcessorGetRBP: mov rax,rbp ret -[extern PostContextSwitch] -[global ArchSwitchContext] ArchSwitchContext: cli mov [gs:16],rcx @@ -804,12 +696,10 @@ ArchSwitchContext: call PostContextSwitch jmp ReturnFromInterruptHandler -[global ProcessorReadCR3] ProcessorReadCR3: mov rax,cr3 ret -[global ProcessorDebugOutputByte] ProcessorDebugOutputByte: %ifdef COM_OUTPUT mov dx,0x3F8 + 5 @@ -824,19 +714,16 @@ ProcessorDebugOutputByte: %endif ret -[global ProcessorReadTimeStamp] ProcessorReadTimeStamp: rdtsc shl rdx,32 or rax,rdx ret -[global ProcessorFlushCodeCache] ProcessorFlushCodeCache: wbinvd ret -[global ProcessorReadMXCSR] ProcessorReadMXCSR: mov rax,.buffer stmxcsr [rax] @@ -845,7 +732,6 @@ ProcessorReadMXCSR: ret .buffer: dq 0 -[global ProcessorInstallTSS] ProcessorInstallTSS: push rbx @@ -875,7 +761,6 @@ ProcessorInstallTSS: pop rbx ret -[global MMArchSafeCopy] MMArchSafeCopy: call GetCurrentThread mov byte [rax + 0],1 ; see definition of Thread @@ -890,7 +775,6 @@ MMArchSafeCopy: mov al,0 ret -[global ProcessorReset] ProcessorReset: in al,0x64 test al,2 @@ -899,8 +783,6 @@ ProcessorReset: out 0x64,al jmp $ -[global _KThreadTerminate] -[extern KThreadTerminate] _KThreadTerminate: sub rsp,8 jmp KThreadTerminate @@ -921,10 +803,8 @@ SynchronizeTimeStampCounter: shr rdx,32 wrmsr ret -[global timeStampCounterSynchronizationValue] timeStampCounterSynchronizationValue: dq 0 -[global ProcessorAPStartup] [bits 16] ProcessorAPStartup: ; This function must be less than 4KB in length (see drivers/acpi.cpp) mov ax,0x1000 @@ -966,14 +846,12 @@ ProcessorAPStartup: ; This function must be less than 4KB in length (see drivers mov rsp,[0x10FD0] call SetupProcessor1 call SynchronizeTimeStampCounter - [extern SetupProcessor2] mov rdi,[0x10FB0] call SetupProcessor2 mov byte [0x10FC0],2 ; Indicate the BSP can start the next processor. and rsp,~0xF jmp ProcessorReady -[global gdt_data] gdt_data: .null_entry: dq 0 .code_entry: dd 0xFFFF ; 0x08 diff --git a/arch/x86_pc.cpp b/arch/x86_pc.cpp new file mode 100644 index 0000000..5641cf1 --- /dev/null +++ b/arch/x86_pc.cpp @@ -0,0 +1,990 @@ +#include + +extern "C" uint64_t ProcessorReadCR3(); + +struct MSIHandler { + KIRQHandler callback; + void *context; +}; + +struct IRQHandler { + KIRQHandler callback; + void *context; + intptr_t line; + KPCIDevice *pciDevice; + const char *cOwnerName; +}; + +uint8_t pciIRQLines[0x100 /* slots */][4 /* pins */]; + +MSIHandler msiHandlers[INTERRUPT_VECTOR_MSI_COUNT]; +IRQHandler irqHandlers[0x40]; +KSpinlock irqHandlersLock; // Also for msiHandlers. + +extern volatile uint64_t timeStampCounterSynchronizationValue; + +PhysicalMemoryRegion *physicalMemoryRegions; +size_t physicalMemoryRegionsCount; +size_t physicalMemoryRegionsPagesCount; +size_t physicalMemoryOriginalPagesCount; +size_t physicalMemoryRegionsIndex; +uintptr_t physicalMemoryHighest; + +uint32_t bootloaderID; +uintptr_t bootloaderInformationOffset; + +// Spinlock since some drivers need to access it in IRQs (e.g. ACPICA). +KSpinlock pciConfigSpinlock; + +KSpinlock ipiLock; + +const char *const exceptionInformation[] = { + "0x00: Divide Error (Fault)", + "0x01: Debug Exception (Fault/Trap)", + "0x02: Non-Maskable External Interrupt (Interrupt)", + "0x03: Breakpoint (Trap)", + "0x04: Overflow (Trap)", + "0x05: BOUND Range Exceeded (Fault)", + "0x06: Invalid Opcode (Fault)", + "0x07: x87 Coprocessor Unavailable (Fault)", + "0x08: Double Fault (Abort)", + "0x09: x87 Coprocessor Segment Overrun (Fault)", + "0x0A: Invalid TSS (Fault)", + "0x0B: Segment Not Present (Fault)", + "0x0C: Stack Protection (Fault)", + "0x0D: General Protection (Fault)", + "0x0E: Page Fault (Fault)", + "0x0F: Reserved/Unknown", + "0x10: x87 FPU Floating-Point Error (Fault)", + "0x11: Alignment Check (Fault)", + "0x12: Machine Check (Abort)", + "0x13: SIMD Floating-Point Exception (Fault)", + "0x14: Virtualization Exception (Fault)", + "0x15: Reserved/Unknown", + "0x16: Reserved/Unknown", + "0x17: Reserved/Unknown", + "0x18: Reserved/Unknown", + "0x19: Reserved/Unknown", + "0x1A: Reserved/Unknown", + "0x1B: Reserved/Unknown", + "0x1C: Reserved/Unknown", + "0x1D: Reserved/Unknown", + "0x1E: Reserved/Unknown", + "0x1F: Reserved/Unknown", +}; + +uint32_t LapicReadRegister(uint32_t reg) { +#ifdef ES_ARCH_X86_64 + return acpi.lapicAddress[reg]; +#else + return ((volatile uint32_t *) LOCAL_APIC_BASE)[reg]; +#endif +} + +void LapicWriteRegister(uint32_t reg, uint32_t value) { +#ifdef ES_ARCH_X86_64 + acpi.lapicAddress[reg] = value; +#else + ((volatile uint32_t *) LOCAL_APIC_BASE)[reg] = value; +#endif +} + +void LapicNextTimer(size_t ms) { + LapicWriteRegister(0x320 >> 2, TIMER_INTERRUPT | (1 << 17)); + LapicWriteRegister(0x380 >> 2, acpi.lapicTicksPerMs * ms); +} + +void LapicEndOfInterrupt() { + LapicWriteRegister(0xB0 >> 2, 0); +} + +uintptr_t MMArchEarlyAllocatePage() { + uintptr_t i = physicalMemoryRegionsIndex; + + while (!physicalMemoryRegions[i].pageCount) { + i++; + + if (i == physicalMemoryRegionsCount) { + KernelPanic("MMArchEarlyAllocatePage - Expected more pages in physical regions.\n"); + } + } + + PhysicalMemoryRegion *region = physicalMemoryRegions + i; + uintptr_t returnValue = region->baseAddress; + + region->baseAddress += K_PAGE_SIZE; + region->pageCount--; + physicalMemoryRegionsPagesCount--; + physicalMemoryRegionsIndex = i; + + return returnValue; +} + +uint64_t MMArchPopulatePageFrameDatabase() { + uint64_t commitLimit = 0; + + for (uintptr_t i = 0; i < physicalMemoryRegionsCount; i++) { + uintptr_t base = physicalMemoryRegions[i].baseAddress >> K_PAGE_BITS; + uintptr_t count = physicalMemoryRegions[i].pageCount; + commitLimit += count; + + for (uintptr_t j = 0; j < count; j++) { + MMPhysicalInsertFreePagesNext(base + j); + } + } + + physicalMemoryRegionsPagesCount = 0; + return commitLimit; +} + +uintptr_t MMArchGetPhysicalMemoryHighest() { + return physicalMemoryHighest; +} + +void ProcessorOut8Delayed(uint16_t port, uint8_t value) { + ProcessorOut8(port, value); + + // Read an unused port to get a short delay. + ProcessorIn8(IO_UNUSED_DELAY); +} + +extern "C" void PCSetupCOM1() { +#ifdef COM_OUTPUT + ProcessorOut8Delayed(IO_COM_1 + 1, 0x00); + ProcessorOut8Delayed(IO_COM_1 + 3, 0x80); + ProcessorOut8Delayed(IO_COM_1 + 0, 0x03); + ProcessorOut8Delayed(IO_COM_1 + 1, 0x00); + ProcessorOut8Delayed(IO_COM_1 + 3, 0x03); + ProcessorOut8Delayed(IO_COM_1 + 2, 0xC7); + ProcessorOut8Delayed(IO_COM_1 + 4, 0x0B); + + // Print a divider line. + for (uint8_t i = 0; i < 10; i++) ProcessorDebugOutputByte('-'); + ProcessorDebugOutputByte('\r'); + ProcessorDebugOutputByte('\n'); +#endif +} + +extern "C" void PCDisablePIC() { + // Remap the ISRs sent by the PIC to 0x20 - 0x2F. + // Even though we'll mask the PIC to use the APIC, + // we have to do this so that the spurious interrupts are sent to a reasonable vector range. + ProcessorOut8Delayed(IO_PIC_1_COMMAND, 0x11); + ProcessorOut8Delayed(IO_PIC_2_COMMAND, 0x11); + ProcessorOut8Delayed(IO_PIC_1_DATA, 0x20); + ProcessorOut8Delayed(IO_PIC_2_DATA, 0x28); + ProcessorOut8Delayed(IO_PIC_1_DATA, 0x04); + ProcessorOut8Delayed(IO_PIC_2_DATA, 0x02); + ProcessorOut8Delayed(IO_PIC_1_DATA, 0x01); + ProcessorOut8Delayed(IO_PIC_2_DATA, 0x01); + + // Mask all interrupts. + ProcessorOut8Delayed(IO_PIC_1_DATA, 0xFF); + ProcessorOut8Delayed(IO_PIC_2_DATA, 0xFF); +} + +extern "C" void PCProcessMemoryMap() { + physicalMemoryRegions = (PhysicalMemoryRegion *) (LOW_MEMORY_MAP_START + 0x60000 + bootloaderInformationOffset); + + for (uintptr_t i = 0; physicalMemoryRegions[i].baseAddress; i++) { + PhysicalMemoryRegion region = physicalMemoryRegions[i]; + uint64_t end = region.baseAddress + (region.pageCount << K_PAGE_BITS); +#ifdef ES_BITS_32 + if (end > 0x100000000) { region.pageCount = 0; continue; } +#endif + physicalMemoryRegionsPagesCount += region.pageCount; + if (end > physicalMemoryHighest) physicalMemoryHighest = end; + physicalMemoryRegionsCount++; + } + + physicalMemoryOriginalPagesCount = physicalMemoryRegions[physicalMemoryRegionsCount].pageCount; +} + +uintptr_t GetBootloaderInformationOffset() { + return bootloaderInformationOffset; +} + +uint32_t KPCIReadConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, int size) { + KSpinlockAcquire(&pciConfigSpinlock); + EsDefer(KSpinlockRelease(&pciConfigSpinlock)); + if (offset & 3) KernelPanic("KPCIReadConfig - offset is not 4-byte aligned."); + ProcessorOut32(IO_PCI_CONFIG, (uint32_t) (0x80000000 | (bus << 16) | (device << 11) | (function << 8) | offset)); + if (size == 8) return ProcessorIn8(IO_PCI_DATA); + if (size == 16) return ProcessorIn16(IO_PCI_DATA); + if (size == 32) return ProcessorIn32(IO_PCI_DATA); + KernelPanic("PCIController::ReadConfig - Invalid size %d.\n", size); + return 0; +} + +void KPCIWriteConfig(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value, int size) { + KSpinlockAcquire(&pciConfigSpinlock); + EsDefer(KSpinlockRelease(&pciConfigSpinlock)); + if (offset & 3) KernelPanic("KPCIWriteConfig - offset is not 4-byte aligned."); + ProcessorOut32(IO_PCI_CONFIG, (uint32_t) (0x80000000 | (bus << 16) | (device << 11) | (function << 8) | offset)); + if (size == 8) ProcessorOut8(IO_PCI_DATA, value); + else if (size == 16) ProcessorOut16(IO_PCI_DATA, value); + else if (size == 32) ProcessorOut32(IO_PCI_DATA, value); + else KernelPanic("PCIController::WriteConfig - Invalid size %d.\n", size); +} + +void MMArchUnmapPages(MMSpace *space, uintptr_t virtualAddressStart, uintptr_t pageCount, unsigned flags, size_t unmapMaximum, uintptr_t *resumePosition) { + // We can't let anyone use the unmapped pages until they've been invalidated on all processors. + // This also synchronises modified bit updating. + KMutexAcquire(&pmm.pageFrameMutex); + EsDefer(KMutexRelease(&pmm.pageFrameMutex)); + + KMutexAcquire(&space->data.mutex); + EsDefer(KMutexRelease(&space->data.mutex)); + +#ifdef ES_ARCH_X86_64 + uintptr_t tableBase = virtualAddressStart & 0x0000FFFFFFFFF000; +#else + uintptr_t tableBase = virtualAddressStart & 0xFFFFF000; +#endif + uintptr_t start = resumePosition ? *resumePosition : 0; + + // TODO Freeing newly empty page tables. + // - What do we need to invalidate when we do this? + + for (uintptr_t i = start; i < pageCount; i++) { + uintptr_t virtualAddress = (i << K_PAGE_BITS) + tableBase; + +#ifdef ES_ARCH_X86_64 + if ((PAGE_TABLE_L4[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3)] & 1) == 0) { + i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 3)); + i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 3)); + continue; + } + + if ((PAGE_TABLE_L3[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2)] & 1) == 0) { + i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 2)); + i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 2)); + continue; + } +#endif + + if ((PAGE_TABLE_L2[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)] & 1) == 0) { + i -= (virtualAddress >> K_PAGE_BITS) % (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 1)); + i += (1 << (ENTRIES_PER_PAGE_TABLE_BITS * 1)); + continue; + } + + uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); + + uintptr_t translation = PAGE_TABLE_L1[indexL1]; + + if (!(translation & 1)) { + // The page wasn't mapped. + continue; + } + + bool copy = translation & (1 << 9); + + if (copy && (flags & MM_UNMAP_PAGES_BALANCE_FILE) && (~flags & MM_UNMAP_PAGES_FREE_COPIED)) { + // Ignore copied pages when balancing file mappings. + continue; + } + + if ((~translation & (1 << 5)) || (~translation & (1 << 6))) { + // See MMArchMapPage for a discussion of why these bits must be set. + KernelPanic("MMArchUnmapPages - Page found without accessed or dirty bit set (virtualAddress: %x, translation: %x).\n", + virtualAddress, translation); + } + + PAGE_TABLE_L1[indexL1] = 0; + +#ifdef ES_ARCH_X86_64 + uintptr_t physicalAddress = translation & 0x0000FFFFFFFFF000; +#else + uintptr_t physicalAddress = translation & 0xFFFFF000; +#endif + + if ((flags & MM_UNMAP_PAGES_FREE) || ((flags & MM_UNMAP_PAGES_FREE_COPIED) && copy)) { + MMPhysicalFree(physicalAddress, true); + } else if (flags & MM_UNMAP_PAGES_BALANCE_FILE) { + // It's safe to do this before page invalidation, + // because the page fault handler is synchronised with the same mutexes acquired above. + + if (MMUnmapFilePage(physicalAddress >> K_PAGE_BITS)) { + if (resumePosition) { + if (!unmapMaximum--) { + *resumePosition = i; + break; + } + } + } + } + } + + MMArchInvalidatePages(virtualAddressStart, pageCount); +} + +bool MMArchMapPage(MMSpace *space, uintptr_t physicalAddress, uintptr_t virtualAddress, unsigned flags) { + // TODO Use the no-execute bit. + + if ((physicalAddress | virtualAddress) & (K_PAGE_SIZE - 1)) { + KernelPanic("MMArchMapPage - Address not page aligned.\n"); + } + + if (pmm.pageFrames && (physicalAddress >> K_PAGE_BITS) < pmm.pageFrameDatabaseCount) { + if (pmm.pageFrames[physicalAddress >> K_PAGE_BITS].state != MMPageFrame::ACTIVE + && pmm.pageFrames[physicalAddress >> K_PAGE_BITS].state != MMPageFrame::UNUSABLE) { + KernelPanic("MMArchMapPage - Physical page frame %x not marked as ACTIVE or UNUSABLE.\n", physicalAddress); + } + } + + if (!physicalAddress) { + KernelPanic("MMArchMapPage - Attempt to map physical page 0.\n"); + } else if (!virtualAddress) { + KernelPanic("MMArchMapPage - Attempt to map virtual page 0.\n"); +#ifdef ES_ARCH_X86_64 + } else if (virtualAddress < 0xFFFF800000000000 && ProcessorReadCR3() != space->data.cr3) { +#else + } else if (virtualAddress < 0xC0000000 && ProcessorReadCR3() != space->data.cr3) { +#endif + KernelPanic("MMArchMapPage - Attempt to map page into other address space.\n"); + } + + bool acquireFrameLock = !(flags & (MM_MAP_PAGE_NO_NEW_TABLES | MM_MAP_PAGE_FRAME_LOCK_ACQUIRED)); + if (acquireFrameLock) KMutexAcquire(&pmm.pageFrameMutex); + EsDefer(if (acquireFrameLock) KMutexRelease(&pmm.pageFrameMutex);); + + bool acquireSpaceLock = ~flags & MM_MAP_PAGE_NO_NEW_TABLES; + if (acquireSpaceLock) KMutexAcquire(&space->data.mutex); + EsDefer(if (acquireSpaceLock) KMutexRelease(&space->data.mutex)); + + // EsPrint("\tMap, %x -> %x\n", virtualAddress, physicalAddress); + + uintptr_t oldVirtualAddress = virtualAddress; +#ifdef ES_ARCH_X86_64 + physicalAddress &= 0xFFFFFFFFFFFFF000; + virtualAddress &= 0x0000FFFFFFFFF000; +#endif + +#ifdef ES_ARCH_X86_64 + uintptr_t indexL4 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3); + uintptr_t indexL3 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2); +#endif + uintptr_t indexL2 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); + + if (space != coreMMSpace && space != kernelMMSpace /* Don't check the kernel's space since the bootloader's tables won't be committed. */) { +#ifdef ES_ARCH_X86_64 + if (!(space->data.l3Commit[indexL4 >> 3] & (1 << (indexL4 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L3 page table.\n"); + if (!(space->data.l2Commit[indexL3 >> 3] & (1 << (indexL3 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L2 page table.\n"); +#endif + if (!(space->data.l1Commit[indexL2 >> 3] & (1 << (indexL2 & 7)))) KernelPanic("MMArchMapPage - Attempt to map using uncommitted L1 page table.\n"); + } + +#ifdef ES_ARCH_X86_64 + if ((PAGE_TABLE_L4[indexL4] & 1) == 0) { + if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); + PAGE_TABLE_L4[indexL4] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; + ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L3 + indexL3)); // Not strictly necessary. + EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L3 + indexL3) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); + space->data.pageTablesActive++; + } + + if ((PAGE_TABLE_L3[indexL3] & 1) == 0) { + if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); + PAGE_TABLE_L3[indexL3] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; + ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L2 + indexL2)); // Not strictly necessary. + EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L2 + indexL2) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); + space->data.pageTablesActive++; + } +#endif + + if ((PAGE_TABLE_L2[indexL2] & 1) == 0) { + if (flags & MM_MAP_PAGE_NO_NEW_TABLES) KernelPanic("MMArchMapPage - NO_NEW_TABLES flag set, but a table was missing.\n"); + PAGE_TABLE_L2[indexL2] = MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_LOCK_ACQUIRED) | 7; + ProcessorInvalidatePage((uintptr_t) (PAGE_TABLE_L1 + indexL1)); // Not strictly necessary. + EsMemoryZero((void *) ((uintptr_t) (PAGE_TABLE_L1 + indexL1) & ~(K_PAGE_SIZE - 1)), K_PAGE_SIZE); + space->data.pageTablesActive++; + } + + uintptr_t oldValue = PAGE_TABLE_L1[indexL1]; + uintptr_t value = physicalAddress | 3; + +#ifdef ES_ARCH_X86_64 + if (flags & MM_MAP_PAGE_WRITE_COMBINING) value |= 16; // This only works because we modified the PAT in SetupProcessor1. +#else + if (flags & MM_MAP_PAGE_WRITE_COMBINING) KernelPanic("MMArchMapPage - Write combining is unimplemented.\n"); // TODO. +#endif + if (flags & MM_MAP_PAGE_NOT_CACHEABLE) value |= 24; + if (flags & MM_MAP_PAGE_USER) value |= 7; + else value |= 1 << 8; // Global. + if (flags & MM_MAP_PAGE_READ_ONLY) value &= ~2; + if (flags & MM_MAP_PAGE_COPIED) value |= 1 << 9; // Ignored by the CPU. + + // When the CPU accesses or writes to a page, + // it will modify the table entry to set the accessed or dirty bits respectively, + // but it uses its TLB entry as the assumed previous value of the entry. + // When unmapping pages we can't atomically remove an entry and do the TLB shootdown. + // This creates a race condition: + // 1. CPU 0 maps a page table entry. The dirty bit is not set. + // 2. CPU 1 reads from the page. A TLB entry is created with the dirty bit not set. + // 3. CPU 0 unmaps the entry. + // 4. CPU 1 writes to the page. As the TLB entry has the dirty bit cleared, it sets the entry to its cached entry ORed with the dirty bit. + // 5. CPU 0 invalidates the entry. + // That is, CPU 1 didn't realize the page was unmapped when it wrote out its entry, so the page becomes mapped again. + // To prevent this, we mark all pages with the dirty and accessed bits when we initially map them. + // (We don't use these bits for anything, anyway. They're basically useless on SMP systems, as far as I can tell.) + // That said, a CPU won't overwrite and clear a dirty bit when writing out its accessed flag (tested on Qemu); + // see here https://stackoverflow.com/questions/69024372/. + // Tl;dr: if a CPU ever sees an entry without these bits set, it can overwrite the entry with junk whenever it feels like it. + // TODO Should we be marking page tables as dirty/accessed? (Including those made by the 32-bit AND 64-bit bootloader and MMArchInitialise). + // When page table trimming is implemented, we'll probably need to do this. + value |= (1 << 5) | (1 << 6); + + if ((oldValue & 1) && !(flags & MM_MAP_PAGE_OVERWRITE)) { + if (flags & MM_MAP_PAGE_IGNORE_IF_MAPPED) { + return false; + } + + if ((oldValue & ~(K_PAGE_SIZE - 1)) != physicalAddress) { + KernelPanic("MMArchMapPage - Attempt to map %x to %x that has already been mapped to %x.\n", + virtualAddress, physicalAddress, oldValue & (~(K_PAGE_SIZE - 1))); + } + + if (oldValue == value) { + KernelPanic("MMArchMapPage - Attempt to rewrite page translation.\n", + physicalAddress, virtualAddress, oldValue & (K_PAGE_SIZE - 1), value & (K_PAGE_SIZE - 1)); + } else if (!(oldValue & 2) && (value & 2)) { + // The page has become writable. + } else { + KernelPanic("MMArchMapPage - Attempt to change flags mapping %x address %x from %x to %x.\n", + physicalAddress, virtualAddress, oldValue & (K_PAGE_SIZE - 1), value & (K_PAGE_SIZE - 1)); + } + } + + PAGE_TABLE_L1[indexL1] = value; + + // We rely on this page being invalidated on this CPU in some places. + ProcessorInvalidatePage(oldVirtualAddress); + + return true; +} + +bool MMArchMakePageWritable(MMSpace *space, uintptr_t virtualAddress) { + KMutexAcquire(&space->data.mutex); + EsDefer(KMutexRelease(&space->data.mutex)); + +#ifdef ES_ARCH_X86_64 + virtualAddress &= 0x0000FFFFFFFFF000; +#else + virtualAddress &= 0xFFFFF000; +#endif + +#ifdef ES_ARCH_X86_64 + uintptr_t indexL4 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3); + if ((PAGE_TABLE_L4[indexL4] & 1) == 0) return false; + uintptr_t indexL3 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2); + if ((PAGE_TABLE_L3[indexL3] & 1) == 0) return false; +#endif + uintptr_t indexL2 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1); + if ((PAGE_TABLE_L2[indexL2] & 1) == 0) return false; + uintptr_t indexL1 = virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0); + if ((PAGE_TABLE_L1[indexL1] & 1) == 0) return false; + + PAGE_TABLE_L1[indexL1] |= 2; + return true; +} + +void MMArchInitialise() { + coreMMSpace->data.cr3 = kernelMMSpace->data.cr3 = ProcessorReadCR3(); + + mmCoreRegions[0].baseAddress = MM_CORE_SPACE_START; + mmCoreRegions[0].pageCount = MM_CORE_SPACE_SIZE / K_PAGE_SIZE; + +#ifdef ES_ARCH_X86_64 + for (uintptr_t i = 0x100; i < 0x200; i++) { + if (PAGE_TABLE_L4[i] == 0) { + // We don't need to commit anything because the PMM isn't ready yet. + PAGE_TABLE_L4[i] = MMPhysicalAllocate(ES_FLAGS_DEFAULT) | 3; + EsMemoryZero((void *) (PAGE_TABLE_L3 + i * 0x200), K_PAGE_SIZE); + } + } + + coreMMSpace->data.l1Commit = coreL1Commit; + KMutexAcquire(&coreMMSpace->reserveMutex); + kernelMMSpace->data.l1Commit = (uint8_t *) MMReserve(coreMMSpace, L1_COMMIT_SIZE_BYTES, MM_REGION_NORMAL | MM_REGION_NO_COMMIT_TRACKING | MM_REGION_FIXED)->baseAddress; + KMutexRelease(&coreMMSpace->reserveMutex); +#endif +} + +uintptr_t MMArchTranslateAddress(MMSpace *, uintptr_t virtualAddress, bool writeAccess) { + // TODO This mutex will be necessary if we ever remove page tables. + // space->data.mutex.Acquire(); + // EsDefer(space->data.mutex.Release()); + +#ifdef ES_ARCH_X86_64 + virtualAddress &= 0x0000FFFFFFFFF000; + if ((PAGE_TABLE_L4[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 3)] & 1) == 0) return 0; + if ((PAGE_TABLE_L3[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 2)] & 1) == 0) return 0; +#endif + if ((PAGE_TABLE_L2[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 1)] & 1) == 0) return 0; + uintptr_t physicalAddress = PAGE_TABLE_L1[virtualAddress >> (K_PAGE_BITS + ENTRIES_PER_PAGE_TABLE_BITS * 0)]; + if (writeAccess && !(physicalAddress & 2)) return 0; +#ifdef ES_ARCH_X86_64 + return (physicalAddress & 1) ? (physicalAddress & 0x0000FFFFFFFFF000) : 0; +#else + return (physicalAddress & 1) ? (physicalAddress & 0xFFFFF000) : 0; +#endif +} + +uintptr_t ArchFindRootSystemDescriptorPointer() { + uint64_t uefiRSDP = *((uint64_t *) (LOW_MEMORY_MAP_START + GetBootloaderInformationOffset() + 0x7FE8)); + + if (uefiRSDP) { + return uefiRSDP; + } + + PhysicalMemoryRegion searchRegions[2]; + + searchRegions[0].baseAddress = (uintptr_t) (((uint16_t *) LOW_MEMORY_MAP_START)[0x40E] << 4) + LOW_MEMORY_MAP_START; + searchRegions[0].pageCount = 0x400; + searchRegions[1].baseAddress = (uintptr_t) 0xE0000 + LOW_MEMORY_MAP_START; + searchRegions[1].pageCount = 0x20000; + + for (uintptr_t i = 0; i < 2; i++) { + for (uintptr_t address = searchRegions[i].baseAddress; + address < searchRegions[i].baseAddress + searchRegions[i].pageCount; + address += 16) { + RootSystemDescriptorPointer *rsdp = (RootSystemDescriptorPointer *) address; + + if (rsdp->signature != SIGNATURE_RSDP) { + continue; + } + + if (rsdp->revision == 0) { + if (EsMemorySumBytes((uint8_t *) rsdp, 20)) { + continue; + } + + return (uintptr_t) rsdp - LOW_MEMORY_MAP_START; + } else if (rsdp->revision == 2) { + if (EsMemorySumBytes((uint8_t *) rsdp, sizeof(RootSystemDescriptorPointer))) { + continue; + } + + return (uintptr_t) rsdp - LOW_MEMORY_MAP_START; + } + } + } + + return 0; +} + +uint64_t ArchGetTimeFromPITMs() { + // TODO This isn't working on real hardware, but ArchDelay1Ms is? + + // NOTE This will only work if called at least once every 50 ms. + // (The PIT only stores a 16-bit counter, which is depleted every 50 ms.) + + static bool started = false; + static uint64_t cumulative = 0, last = 0; + + if (!started) { + ProcessorOut8(IO_PIT_COMMAND, 0x30); + ProcessorOut8(IO_PIT_DATA, 0xFF); + ProcessorOut8(IO_PIT_DATA, 0xFF); + started = true; + last = 0xFFFF; + return 0; + } else { + ProcessorOut8(IO_PIT_COMMAND, 0x00); + uint16_t x = ProcessorIn8(IO_PIT_DATA); + x |= (ProcessorIn8(IO_PIT_DATA)) << 8; + cumulative += last - x; + if (x > last) cumulative += 0x10000; + last = x; + return cumulative * 1000 / 1193182; + } +} + +void ArchDelay1Ms() { + ProcessorOut8(IO_PIT_COMMAND, 0x30); + ProcessorOut8(IO_PIT_DATA, 0xA9); + ProcessorOut8(IO_PIT_DATA, 0x04); + + while (true) { + ProcessorOut8(IO_PIT_COMMAND, 0xE2); + + if (ProcessorIn8(IO_PIT_DATA) & (1 << 7)) { + break; + } + } +} + +NewProcessorStorage AllocateNewProcessorStorage(ArchCPU *archCPU) { + NewProcessorStorage storage = {}; + storage.local = (CPULocalStorage *) EsHeapAllocate(sizeof(CPULocalStorage), true, K_FIXED); +#ifdef ES_ARCH_X86_64 + storage.gdt = (uint32_t *) MMMapPhysical(kernelMMSpace, MMPhysicalAllocate(MM_PHYSICAL_ALLOCATE_COMMIT_NOW), K_PAGE_SIZE, ES_FLAGS_DEFAULT); +#endif + storage.local->archCPU = archCPU; + archCPU->local = storage.local; + scheduler.CreateProcessorThreads(storage.local); + archCPU->kernelProcessorID = storage.local->processorID; + return storage; +} + +void SetupProcessor2(NewProcessorStorage *storage) { + // Setup the local interrupts for the current processor. + + for (uintptr_t i = 0; i < acpi.lapicNMICount; i++) { + if (acpi.lapicNMIs[i].processor == 0xFF + || acpi.lapicNMIs[i].processor == storage->local->archCPU->processorID) { + uint32_t registerIndex = (0x350 + (acpi.lapicNMIs[i].lintIndex << 4)) >> 2; + uint32_t value = 2 | (1 << 10); // NMI exception interrupt vector. + if (acpi.lapicNMIs[i].activeLow) value |= 1 << 13; + if (acpi.lapicNMIs[i].levelTriggered) value |= 1 << 15; + LapicWriteRegister(registerIndex, value); + } + } + + LapicWriteRegister(0x350 >> 2, LapicReadRegister(0x350 >> 2) & ~(1 << 16)); + LapicWriteRegister(0x360 >> 2, LapicReadRegister(0x360 >> 2) & ~(1 << 16)); + LapicWriteRegister(0x080 >> 2, 0); + if (LapicReadRegister(0x30 >> 2) & 0x80000000) LapicWriteRegister(0x410 >> 2, 0); + LapicEndOfInterrupt(); + + // Configure the LAPIC's timer. + + LapicWriteRegister(0x3E0 >> 2, 2); // Divisor = 16 + + // Create the processor's local storage. + + ProcessorSetLocalStorage(storage->local); + + // Setup a GDT and TSS for the processor. + +#ifdef ES_ARCH_X86_64 + uint32_t *gdt = storage->gdt; + void *bootstrapGDT = (void *) (((uint64_t *) ((uint16_t *) processorGDTR + 1))[0]); + EsMemoryCopy(gdt, bootstrapGDT, 2048); + uint32_t *tss = (uint32_t *) ((uint8_t *) storage->gdt + 2048); + storage->local->archCPU->kernelStack = (void **) (tss + 1); + ProcessorInstallTSS(gdt, tss); +#endif +} + +void ArchInitialise() { + ACPIParseTables(); + + uint8_t bootstrapLapicID = (LapicReadRegister(0x20 >> 2) >> 24); + + ArchCPU *currentCPU = nullptr; + + for (uintptr_t i = 0; i < acpi.processorCount; i++) { + if (acpi.processors[i].apicID == bootstrapLapicID) { + // That's us! + currentCPU = acpi.processors + i; + currentCPU->bootProcessor = true; + break; + } + } + + if (!currentCPU) { + KernelPanic("ArchInitialise - Could not find the bootstrap processor\n"); + } + + // Calibrate the LAPIC's timer and processor's timestamp counter. + ProcessorDisableInterrupts(); + uint64_t start = ProcessorReadTimeStamp(); + LapicWriteRegister(0x380 >> 2, (uint32_t) -1); + for (int i = 0; i < 8; i++) ArchDelay1Ms(); // Average over 8ms + acpi.lapicTicksPerMs = ((uint32_t) -1 - LapicReadRegister(0x390 >> 2)) >> 4; + EsRandomAddEntropy(LapicReadRegister(0x390 >> 2)); + uint64_t end = ProcessorReadTimeStamp(); + timeStampTicksPerMs = (end - start) >> 3; + ProcessorEnableInterrupts(); + // EsPrint("timeStampTicksPerMs = %d\n", timeStampTicksPerMs); + + // Finish processor initialisation. + // This sets up interrupts, the timer, CPULocalStorage, the GDT and TSS, + // and registers the processor with the scheduler. + + NewProcessorStorage storage = AllocateNewProcessorStorage(currentCPU); + SetupProcessor2(&storage); +} + +size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi, int processorID) { + // It's possible that another CPU is trying to send an IPI at the same time we want to send the panic IPI. + // TODO What should we do in this case? + if (interrupt != KERNEL_PANIC_IPI) KSpinlockAssertLocked(&ipiLock); + + // Note: We send IPIs at a special priority that ProcessorDisableInterrupts doesn't mask. + + size_t ignored = 0; + + for (uintptr_t i = 0; i < acpi.processorCount; i++) { + ArchCPU *processor = acpi.processors + i; + + if (processorID != -1) { + if (processorID != processor->kernelProcessorID) { + ignored++; + continue; + } + } else { + if (processor == GetLocalStorage()->archCPU || !processor->local || !processor->local->schedulerReady) { + ignored++; + continue; + } + } + + uint32_t destination = acpi.processors[i].apicID << 24; + uint32_t command = interrupt | (1 << 14) | (nmi ? 0x400 : 0); + LapicWriteRegister(0x310 >> 2, destination); + LapicWriteRegister(0x300 >> 2, command); + + // Wait for the interrupt to be sent. + while (LapicReadRegister(0x300 >> 2) & (1 << 12)); + } + + return ignored; +} + +void ProcessorSendYieldIPI(Thread *thread) { + thread->receivedYieldIPI = false; + KSpinlockAcquire(&ipiLock); + ProcessorSendIPI(YIELD_IPI, false); + KSpinlockRelease(&ipiLock); + while (!thread->receivedYieldIPI); // Spin until the thread gets the IPI. +} + +void ArchNextTimer(size_t ms) { + while (!scheduler.started); // Wait until the scheduler is ready. + GetLocalStorage()->schedulerReady = true; // Make sure this CPU can be scheduled. + LapicNextTimer(ms); // Set the next timer. +} + +uint64_t ArchGetTimeMs() { + // Update the time stamp counter synchronization value. + timeStampCounterSynchronizationValue = ((timeStampCounterSynchronizationValue & 0x8000000000000000) + ^ 0x8000000000000000) | ProcessorReadTimeStamp(); + +#ifdef ES_ARCH_X86_64 + if (acpi.hpetBaseAddress && acpi.hpetPeriod) { + __int128 fsToMs = 1000000000000; + __int128 reading = acpi.hpetBaseAddress[30]; + return (uint64_t) (reading * (__int128) acpi.hpetPeriod / fsToMs); + } +#endif + + return ArchGetTimeFromPITMs(); +} + +extern "C" bool PostContextSwitch(InterruptContext *context, MMSpace *oldAddressSpace) { + Thread *currentThread = GetCurrentThread(); + +#ifdef ES_ARCH_X86_64 + CPULocalStorage *local = GetLocalStorage(); + void *kernelStack = (void *) currentThread->kernelStack; + *local->archCPU->kernelStack = kernelStack; +#endif + + bool newThread = currentThread->cpuTimeSlices == 1; + + LapicEndOfInterrupt(); + ContextSanityCheck(context); + + if (ProcessorAreInterruptsEnabled()) { + KernelPanic("PostContextSwitch - Interrupts were enabled. (1)\n"); + } + +#ifdef ES_ARCH_X86_64 + KernelLog(LOG_VERBOSE, "Arch", "context switch", "Context switch to %zthread %x at %x\n", newThread ? "new " : "", currentThread, context->rip); + currentThread->lastKnownExecutionAddress = context->rip; +#else + KernelLog(LOG_VERBOSE, "Arch", "context switch", "Context switch to %zthread %x at %x\n", newThread ? "new " : "", currentThread, context->eip); + currentThread->lastKnownExecutionAddress = context->eip; +#endif + + if (scheduler.lock.interruptsEnabled) { + KernelPanic("PostContextSwitch - Interrupts were enabled. (3)\n"); + } + + ProcessorSetThreadStorage(currentThread->tlsAddress); + + // We can only free the scheduler's spinlock when we are no longer using the stack + // from the previous thread. See DoContextSwitch. + // (Another CPU can KillThread this once it's back in activeThreads.) + KSpinlockRelease(&scheduler.lock, true); + + if (ProcessorAreInterruptsEnabled()) { + KernelPanic("PostContextSwitch - Interrupts were enabled. (2)\n"); + } + + MMSpaceCloseReference(oldAddressSpace); + +#ifdef ES_ARCH_X86_32 + if (context->fromRing0) { + // Returning to a kernel thread; we need to fix the stack. + uint32_t irq = context->esp; + uint32_t errorCode = context->ss; + context->ss = context->flags; + context->esp = context->cs; + context->flags = context->eip; + context->cs = context->errorCode; + context->eip = context->irq; + context->irq = irq; + context->errorCode = errorCode; + } +#endif + + return newThread; +} + +bool SetupInterruptRedirectionEntry(uintptr_t _line) { + KSpinlockAssertLocked(&scheduler.lock); + + static uint32_t alreadySetup = 0; + + if (alreadySetup & (1 << _line)) { + return true; + } + + // Work out which interrupt the IoApic will sent to the processor. + // TODO Use the upper 4 bits for IRQ priority. + + uintptr_t line = _line; + uintptr_t thisProcessorIRQ = line + IRQ_BASE; + + bool activeLow = false; + bool levelTriggered = true; + + // If there was an interrupt override entry in the MADT table, + // then we'll have to use that number instead. + + for (uintptr_t i = 0; i < acpi.interruptOverrideCount; i++) { + ACPIInterruptOverride *interruptOverride = acpi.interruptOverrides + i; + + if (interruptOverride->sourceIRQ == line) { + line = interruptOverride->gsiNumber; + activeLow = interruptOverride->activeLow; + levelTriggered = interruptOverride->levelTriggered; + break; + } + } + + KernelLog(LOG_INFO, "Arch", "IRQ flags", "SetupInterruptRedirectionEntry - IRQ %d is active %z, %z triggered.\n", + line, activeLow ? "low" : "high", levelTriggered ? "level" : "edge"); + + ACPIIoApic *ioApic; + bool foundIoApic = false; + + // Look for the IoApic to which this interrupt is sent. + + for (uintptr_t i = 0; i < acpi.ioapicCount; i++) { + ioApic = acpi.ioApics + i; + if (line >= ioApic->gsiBase && line < (ioApic->gsiBase + (0xFF & (ACPIIoApicReadRegister(ioApic, 1) >> 16)))) { + foundIoApic = true; + line -= ioApic->gsiBase; + break; + } + } + + // We couldn't find the IoApic that handles this interrupt. + + if (!foundIoApic) { + KernelLog(LOG_ERROR, "Arch", "no IOAPIC", "SetupInterruptRedirectionEntry - Could not find an IOAPIC handling interrupt line %d.\n", line); + return false; + } + + // A normal priority interrupt. + + uintptr_t redirectionTableIndex = line * 2 + 0x10; + uint32_t redirectionEntry = thisProcessorIRQ; + if (activeLow) redirectionEntry |= (1 << 13); + if (levelTriggered) redirectionEntry |= (1 << 15); + + // Send the interrupt to the processor that registered the interrupt. + + ACPIIoApicWriteRegister(ioApic, redirectionTableIndex, 1 << 16); // Mask the interrupt while we modify the entry. + ACPIIoApicWriteRegister(ioApic, redirectionTableIndex + 1, GetLocalStorage()->archCPU->apicID << 24); + ACPIIoApicWriteRegister(ioApic, redirectionTableIndex, redirectionEntry); + + alreadySetup |= 1 << _line; + return true; +} + +void KUnregisterMSI(uintptr_t tag) { + KSpinlockAcquire(&irqHandlersLock); + EsDefer(KSpinlockRelease(&irqHandlersLock)); + msiHandlers[tag].callback = nullptr; +} + +KMSIInformation KRegisterMSI(KIRQHandler handler, void *context, const char *cOwnerName) { + KSpinlockAcquire(&irqHandlersLock); + EsDefer(KSpinlockRelease(&irqHandlersLock)); + + for (uintptr_t i = 0; i < INTERRUPT_VECTOR_MSI_COUNT; i++) { + if (msiHandlers[i].callback) continue; + msiHandlers[i] = { handler, context }; + + // TODO Selecting the best target processor. + // Currently this sends everything to processor 0. + + KernelLog(LOG_INFO, "Arch", "register MSI", "Register MSI with vector %X for '%z'.\n", + INTERRUPT_VECTOR_MSI_START + i, cOwnerName); + + return { + .address = 0xFEE00000, + .data = INTERRUPT_VECTOR_MSI_START + i, + .tag = i, + }; + } + + return {}; +} + +bool KRegisterIRQ(intptr_t line, KIRQHandler handler, void *context, const char *cOwnerName, KPCIDevice *pciDevice) { + KSpinlockAcquire(&scheduler.lock); + EsDefer(KSpinlockRelease(&scheduler.lock)); + + if (line == -1 && !pciDevice) { + KernelPanic("KRegisterIRQ - Interrupt line is %d, and pciDevice is %x.\n", line, pciDevice); + } + + // Save the handler callback and context. + + if (line > 0x20 || line < -1) KernelPanic("KRegisterIRQ - Unexpected IRQ %d\n", line); + bool found = false; + + KSpinlockAcquire(&irqHandlersLock); + + for (uintptr_t i = 0; i < sizeof(irqHandlers) / sizeof(irqHandlers[0]); i++) { + if (!irqHandlers[i].callback) { + found = true; + irqHandlers[i].callback = handler; + irqHandlers[i].context = context; + irqHandlers[i].line = line; + irqHandlers[i].pciDevice = pciDevice; + irqHandlers[i].cOwnerName = cOwnerName; + break; + } + } + + KSpinlockRelease(&irqHandlersLock); + + if (!found) { + KernelLog(LOG_ERROR, "Arch", "too many IRQ handlers", "The limit of IRQ handlers was reached (%d), and the handler for '%z' was not registered.\n", + sizeof(irqHandlers) / sizeof(irqHandlers[0]), cOwnerName); + return false; + } + + KernelLog(LOG_INFO, "Arch", "register IRQ", "KRegisterIRQ - Registered IRQ %d to '%z'.\n", line, cOwnerName); + + if (line != -1) { + if (!SetupInterruptRedirectionEntry(line)) { + return false; + } + } else { + SetupInterruptRedirectionEntry(9); + SetupInterruptRedirectionEntry(10); + SetupInterruptRedirectionEntry(11); + } + + return true; +} diff --git a/arch/x86_pc.h b/arch/x86_pc.h index e152e9d..2a04f5a 100644 --- a/arch/x86_pc.h +++ b/arch/x86_pc.h @@ -17,6 +17,7 @@ #define IO_PS2_COMMAND (0x0064) #define IO_RTC_INDEX (0x0070) #define IO_RTC_DATA (0x0071) +#define IO_UNUSED_DELAY (0x0080) #define IO_PIC_2_COMMAND (0x00A0) #define IO_PIC_2_DATA (0x00A1) #define IO_BGA_INDEX (0x01CE) @@ -47,9 +48,51 @@ #define IO_PCI_CONFIG (0x0CF8) #define IO_PCI_DATA (0x0CFC) +// --------------------------------- Interrupt vectors. + +// Interrupt vectors: +// 0x00 - 0x1F: CPU exceptions +// 0x20 - 0x2F: PIC (disabled, spurious) +// 0x30 - 0x4F: Timers and low-priority IPIs. +// 0x50 - 0x6F: APIC (standard) +// 0x70 - 0xAF: MSI +// 0xF0 - 0xFE: High-priority IPIs +// 0xFF: APIC (spurious interrupt) + +#define TIMER_INTERRUPT (0x40) +#define YIELD_IPI (0x41) +#define IRQ_BASE (0x50) +#define CALL_FUNCTION_ON_ALL_PROCESSORS_IPI (0xF0) +#define TLB_SHOOTDOWN_IPI (0xF1) +#define KERNEL_PANIC_IPI (0) // NMIs ignore the interrupt vector. + +#define INTERRUPT_VECTOR_MSI_START (0x70) +#define INTERRUPT_VECTOR_MSI_COUNT (0x40) + // --------------------------------- Forward declarations. +struct NewProcessorStorage { + struct CPULocalStorage *local; + uint32_t *gdt; +}; + uint8_t ACPIGetCenturyRegisterIndex(); uintptr_t GetBootloaderInformationOffset(); +extern "C" void ProcessorDebugOutputByte(uint8_t byte); +extern uintptr_t bootloaderInformationOffset; +uintptr_t ArchFindRootSystemDescriptorPointer(); +void ArchStartupApplicationProcessors(); +uint32_t LapicReadRegister(uint32_t reg); +void LapicWriteRegister(uint32_t reg, uint32_t value); +NewProcessorStorage AllocateNewProcessorStorage(struct ArchCPU *archCPU); +extern "C" void SetupProcessor2(struct NewProcessorStorage *); +void ArchDelay1Ms(); // Spin for approximately 1ms. Use only during initialisation. Not thread-safe. +uint64_t ArchGetTimeFromPITMs(); +void *ACPIGetRSDP(); +size_t ProcessorSendIPI(uintptr_t interrupt, bool nmi = false, int processorID = -1); // Returns the number of processors the IPI was *not* sent to. +void ArchSetPCIIRQLine(uint8_t slot, uint8_t pin, uint8_t line); +extern "C" void ProcessorReset(); +void MMArchInvalidatePages(uintptr_t virtualAddressStart, uintptr_t pageCount); +void ContextSanityCheck(struct InterruptContext *context); #endif diff --git a/drivers/acpi.cpp b/drivers/acpi.cpp index edbb35a..4a07e01 100644 --- a/drivers/acpi.cpp +++ b/drivers/acpi.cpp @@ -245,7 +245,9 @@ void ACPIParseTables() { uintptr_t startLength = length; uint8_t *data = (uint8_t *) (madt + 1); +#ifdef ES_ARCH_X86_64 acpi.lapicAddress = (uint32_t volatile *) ACPIMapPhysicalMemory(madt->lapicAddress, 0x10000); +#endif while (length && length <= startLength) { uint8_t entryType = data[0]; diff --git a/drivers/ahci.cpp b/drivers/ahci.cpp index f8e92a0..ebf9167 100644 --- a/drivers/ahci.cpp +++ b/drivers/ahci.cpp @@ -281,7 +281,11 @@ bool AHCIController::Access(uintptr_t portIndex, uint64_t offsetBytes, size_t co KDMASegment segment = KDMABufferNextSegment(buffer); prdt[0 + 4 * prdtEntryCount] = segment.physicalAddress; +#ifdef ES_BITS_64 prdt[1 + 4 * prdtEntryCount] = segment.physicalAddress >> 32; +#else + prdt[1 + 4 * prdtEntryCount] = 0; +#endif prdt[2 + 4 * prdtEntryCount] = 0; prdt[3 + 4 * prdtEntryCount] = (segment.byteCount - 1) | (segment.isLast ? (1 << 31) /* IRQ when done */ : 0); @@ -572,9 +576,14 @@ void AHCIController::Initialise() { // Set the registers to the physical addresses. WR_REGISTER_PCLB(i, physicalAddress); - if (dma64Supported) WR_REGISTER_PCLBU(i, physicalAddress >> 32); WR_REGISTER_PFB(i, (physicalAddress + 0x400)); +#ifdef ES_BITS_64 + if (dma64Supported) WR_REGISTER_PCLBU(i, physicalAddress >> 32); if (dma64Supported) WR_REGISTER_PFBU(i, (physicalAddress + 0x400) >> 32); +#else + if (dma64Supported) WR_REGISTER_PCLBU(i, 0); + if (dma64Supported) WR_REGISTER_PFBU(i, 0); +#endif // Point each command list entry to the corresponding command table. @@ -583,7 +592,11 @@ void AHCIController::Initialise() { for (uintptr_t j = 0; j < commandSlotCount; j++) { uintptr_t address = physicalAddress + COMMAND_LIST_SIZE + RECEIVED_FIS_SIZE + COMMAND_TABLE_SIZE * j; commandList[j * 8 + 2] = address; +#ifdef ES_BITS_64 commandList[j * 8 + 3] = address >> 32; +#else + commandList[j * 8 + 3] = 0; +#endif } // Reset the port. @@ -725,7 +738,11 @@ void AHCIController::Initialise() { uint32_t *prdt = (uint32_t *) (ports[i].commandTables + 0x80); prdt[0] = identifyDataPhysical; +#ifdef ES_BITS_64 prdt[1] = identifyDataPhysical >> 32; +#else + prdt[1] = 0; +#endif prdt[2] = 0; prdt[3] = 0x200 - 1; diff --git a/drivers/esfs2.cpp b/drivers/esfs2.cpp index 9e3d0c6..7067917 100644 --- a/drivers/esfs2.cpp +++ b/drivers/esfs2.cpp @@ -105,7 +105,7 @@ static EsError FindDirectoryEntryReferenceFromIndex(Volume *volume, uint8_t *buf } } -static bool ValidateDirectoryEntry(DirectoryEntry *entry) { +static bool ValidateDirectoryEntry(Volume *volume, DirectoryEntry *entry) { uint32_t checksum = entry->checksum; entry->checksum = 0; uint32_t calculated = CalculateCRC32(entry, sizeof(DirectoryEntry)); @@ -129,6 +129,8 @@ static bool ValidateDirectoryEntry(DirectoryEntry *entry) { ESFS_CHECK(data->count == entry->fileSize, "ValidateDirectoryEntry - Expected direct attribute to cover entire file."); } } else if (attribute->type == ESFS_ATTRIBUTE_DIRECTORY) { + AttributeDirectory *directory = (AttributeDirectory *) attribute; + ESFS_CHECK(directory->indexRootBlock < volume->superblock.blockCount, "ValidateDirectoryEntry - Directory index root block outside volume."); } else if (attribute->type == ESFS_ATTRIBUTE_FILENAME) { AttributeFilename *filename = (AttributeFilename *) attribute; ESFS_CHECK(filename->length + 8 <= filename->size, "ValidateDirectoryEntry - Filename too long."); @@ -350,7 +352,7 @@ static void Sync(KNode *_directory, KNode *node) { return; } - if (!ValidateDirectoryEntry((DirectoryEntry *) (blockBuffer + file->reference.offsetIntoBlock))) { + if (!ValidateDirectoryEntry(volume, (DirectoryEntry *) (blockBuffer + file->reference.offsetIntoBlock))) { return; } @@ -393,7 +395,7 @@ static EsError Enumerate(KNode *node) { for (uint64_t j = 0; j < entriesInThisBlock; j++, reference.offsetIntoBlock += sizeof(DirectoryEntry)) { DirectoryEntry *entry = (DirectoryEntry *) blockBuffer + j; - if (!ValidateDirectoryEntry(entry)) { + if (!ValidateDirectoryEntry(volume, entry)) { // Try the entries in the next block. break; } @@ -1635,7 +1637,7 @@ static bool CreateInternal(const char *name, size_t nameLength, EsNodeType type, entry->checksum = 0; entry->checksum = CalculateCRC32(entry, sizeof(DirectoryEntry)); - if (!ValidateDirectoryEntry(entry)) KernelPanic("EsFS::CreateInternal - Created directory entry is invalid.\n"); + if (!ValidateDirectoryEntry(volume, entry)) KernelPanic("EsFS::CreateInternal - Created directory entry is invalid.\n"); // Write the directory entry. @@ -1670,7 +1672,7 @@ static EsError Move(KNode *_oldDirectory, KNode *_file, KNode *_newDirectory, co file->entry.checksum = 0; file->entry.checksum = CalculateCRC32(&file->entry, sizeof(DirectoryEntry)); - if (!ValidateDirectoryEntry(&file->entry)) KernelPanic("EsFS::Move - Existing entry is invalid.\n"); + if (!ValidateDirectoryEntry(volume, &file->entry)) KernelPanic("EsFS::Move - Existing entry is invalid.\n"); uint8_t *buffers = (uint8_t *) EsHeapAllocate(superblock->blockSize * 2, true, K_FIXED); if (!buffers) return ES_ERROR_INSUFFICIENT_RESOURCES; @@ -1727,7 +1729,7 @@ static EsError Load(KNode *_directory, KNode *_node, KNodeMetadata *, const void DirectoryEntry *entry = (DirectoryEntry *) (blockBuffer + reference.offsetIntoBlock); - if (!ValidateDirectoryEntry(entry)) { + if (!ValidateDirectoryEntry(directory->volume, entry)) { return ES_ERROR_CORRUPT_DATA; } @@ -1788,7 +1790,7 @@ static EsError Scan(const char *name, size_t nameLength, KNode *_directory) { } DirectoryEntry *entry = (DirectoryEntry *) (blockBuffer + reference.offsetIntoBlock); - if (!ValidateDirectoryEntry(entry)) return ES_ERROR_CORRUPT_DATA; + if (!ValidateDirectoryEntry(volume, entry)) return ES_ERROR_CORRUPT_DATA; if ((entry->nodeType == ESFS_NODE_TYPE_DIRECTORY && !FindAttribute(entry, ESFS_ATTRIBUTE_DIRECTORY)) || (entry->nodeType == ESFS_NODE_TYPE_FILE && !FindAttribute(entry, ESFS_ATTRIBUTE_DATA))) { @@ -1907,7 +1909,7 @@ static bool Mount(Volume *volume, EsFileOffsetDifference *rootDirectoryChildren) } DirectoryEntry *entry = (DirectoryEntry *) (blockBuffer + rootReference.offsetIntoBlock); - if (!ValidateDirectoryEntry(entry)) goto failure; + if (!ValidateDirectoryEntry(volume, entry)) goto failure; AttributeDirectory *directory = (AttributeDirectory *) FindAttribute(entry, ESFS_ATTRIBUTE_DIRECTORY); if (!directory || !FindAttribute(entry, ESFS_ATTRIBUTE_DATA)) { diff --git a/drivers/nvme.cpp b/drivers/nvme.cpp index e496c23..c8be4cc 100644 --- a/drivers/nvme.cpp +++ b/drivers/nvme.cpp @@ -477,7 +477,7 @@ void NVMeController::Initialise() { return; } - if (~capabilities & (1UL << 37)) { + if (~capabilities & (1ULL << 37)) { KernelLog(LOG_ERROR, "NVMe", "unsupported capabilities", "Controller does not support NVMe command set.\n"); return; } diff --git a/drivers/pci.cpp b/drivers/pci.cpp index ce56036..6483258 100644 --- a/drivers/pci.cpp +++ b/drivers/pci.cpp @@ -287,7 +287,11 @@ bool KPCIDevice::EnableMSI(KIRQHandler irqHandler, void *context, const char *cO WriteConfig32(pointer + 4, msi.address & 0xFFFFFFFF); if (control & (1 << 7)) { +#ifdef ES_BITS_64 WriteConfig32(pointer + 8, msi.address >> 32); +#else + WriteConfig32(pointer + 8, 0); +#endif WriteConfig16(pointer + 12, (ReadConfig16(pointer + 12) & 0x3800) | msi.data); if (control & (1 << 8)) WriteConfig32(pointer + 16, 0); } else { diff --git a/drivers/xhci.cpp b/drivers/xhci.cpp index 371cced..31b3d06 100644 --- a/drivers/xhci.cpp +++ b/drivers/xhci.cpp @@ -596,7 +596,7 @@ bool XHCIController::HandleIRQ() { uint8_t completionCode = (dw2 >> 24) & 0xFF; KernelLog(LOG_VERBOSE, "xHCI", "got event", "Received event of type %d with code %d from %x.\n", - type, completionCode, (uintptr_t) dw0 | ((uintptr_t) dw1 << 32)); + type, completionCode, (uint64_t) dw0 | ((uint64_t) dw1 << 32)); if (type == 32 /* transfer completion event */) { uint8_t slotID = (dw3 >> 24) & 0xFF; diff --git a/kernel/kernel.h b/kernel/kernel.h index 367ecfe..8250bb9 100644 --- a/kernel/kernel.h +++ b/kernel/kernel.h @@ -131,7 +131,6 @@ extern "C" { void MMArchUnmapPages(MMSpace *space, uintptr_t virtualAddressStart, uintptr_t pageCount, unsigned flags, size_t unmapMaximum = 0, uintptr_t *resumePosition = nullptr); bool MMArchMakePageWritable(MMSpace *space, uintptr_t virtualAddress); bool MMArchHandlePageFault(uintptr_t address, uint32_t flags); - void MMArchInvalidatePages(uintptr_t virtualAddressStart, uintptr_t pageCount); bool MMArchIsBufferInUserRange(uintptr_t baseAddress, size_t byteCount); bool MMArchSafeCopy(uintptr_t destinationAddress, uintptr_t sourceAddress, size_t byteCount); // Returns false if a page fault occured during the copy. bool MMArchCommitPageTables(MMSpace *space, struct MMRegion *region); diff --git a/kernel/memory.cpp b/kernel/memory.cpp index 3c70803..55ebe60 100644 --- a/kernel/memory.cpp +++ b/kernel/memory.cpp @@ -2282,10 +2282,10 @@ void MMInitialise() { KMutexRelease(&kernelMMSpace->reserveMutex); // 1 extra for the top page, then round up so the page bitset is byte-aligned. - pmm.pageFrameDatabaseCount = (MMArchGetPhysicalMemoryHighest() + (K_PAGE_SIZE << 3)) >> K_PAGE_BITS; - - pmm.pageFrames = (MMPageFrame *) MMStandardAllocate(kernelMMSpace, pmm.pageFrameDatabaseCount * sizeof(MMPageFrame), MM_REGION_FIXED); - pmm.freeOrZeroedPageBitset.Initialise(pmm.pageFrameDatabaseCount, true); + uintptr_t pageFrameDatabaseCount = (MMArchGetPhysicalMemoryHighest() + (K_PAGE_SIZE << 3)) >> K_PAGE_BITS; + pmm.pageFrames = (MMPageFrame *) MMStandardAllocate(kernelMMSpace, pageFrameDatabaseCount * sizeof(MMPageFrame), MM_REGION_FIXED); + pmm.freeOrZeroedPageBitset.Initialise(pageFrameDatabaseCount, true); + pmm.pageFrameDatabaseCount = pageFrameDatabaseCount; // Only set this after the database is ready, or it may be accessed mid-allocation! MMPhysicalInsertFreePagesStart(); uint64_t commitLimit = MMArchPopulatePageFrameDatabase(); diff --git a/kernel/scheduler.cpp b/kernel/scheduler.cpp index d871c4b..0470a45 100644 --- a/kernel/scheduler.cpp +++ b/kernel/scheduler.cpp @@ -396,7 +396,11 @@ Thread *Scheduler::SpawnThread(const char *cName, uintptr_t startAddress, uintpt thread->handles = 2; // Allocate the thread's stacks. +#if defined(ES_BITS_64) uintptr_t kernelStackSize = userland ? 0x4000 /* 16KB */ : 0x10000 /* 64KB */; +#elif defined(ES_BITS_32) + uintptr_t kernelStackSize = userland ? 0x3000 /* 12KB */ : 0x8000 /* 32KB */; +#endif uintptr_t userStackReserve = userland ? 0x400000 /* 4MB */ : kernelStackSize; uintptr_t userStackCommit = userland ? 0x20000 /* 128KB */ : 0; uintptr_t stack = 0, kernelStack = (uintptr_t) MMStandardAllocate(kernelMMSpace, kernelStackSize, MM_REGION_FIXED); @@ -781,6 +785,7 @@ void Thread::SetAddressSpace(MMSpace *space) { KSpinlockAcquire(&scheduler.lock); MMSpace *oldSpace = temporaryAddressSpace ?: kernelMMSpace; + EsPrint("space = %x, oldSpace = %x\n", space, oldSpace); temporaryAddressSpace = space; MMSpace *newSpace = space ?: kernelMMSpace; MMSpaceOpenReference(newSpace); diff --git a/kernel/terminal.cpp b/kernel/terminal.cpp index 40c54e9..f924d74 100644 --- a/kernel/terminal.cpp +++ b/kernel/terminal.cpp @@ -16,7 +16,6 @@ KMutex printLock; #endif void DebugWriteCharacter(uintptr_t character); -extern "C" void ProcessorDebugOutputByte(uint8_t byte); int KWaitKey(); #if defined(ES_ARCH_X86_32) || defined(ES_ARCH_X86_64) diff --git a/shared/esfs2.h b/shared/esfs2.h index d7b47fa..212ca43 100644 --- a/shared/esfs2.h +++ b/shared/esfs2.h @@ -74,6 +74,7 @@ typedef struct AttributeFilename { typedef struct AttributeDirectory { /* 0 */ uint16_t type; // ESFS_ATTRIBUTE_DIRECTORY. /* 2 */ uint16_t size; // The size in bytes. Must be 8 byte aligned. + /* 4 */ uint8_t _unused0[4]; /* 8 */ uint64_t childNodes; // The number of child nodes in the directory. /* 16 */ uint64_t indexRootBlock; // The block containing the root IndexVertex for the directory. /* 24 */ uint64_t totalSize; // The sum of sizes of all the directory's children in bytes. @@ -157,12 +158,14 @@ typedef struct Superblock { /* 52 */ uint32_t checksum; // CRC-32 checksum of Superblock. /* 56 */ uint8_t mounted; // Non-zero to indicate that the volume is mounted, or was not properly unmounted. + /* 57 */ uint8_t _unused2[7]; /* 64 */ uint64_t blockSize; // The size of a block on the volume. /* 72 */ uint64_t blockCount; // The number of blocks on the volume. /* 80 */ uint64_t blocksUsed; // The number of blocks that are in use. /* 88 */ uint32_t blocksPerGroup; // The number of blocks in a group. + /* 92 */ uint8_t _unused3[4]; /* 96 */ uint64_t groupCount; // The number of groups on the volume. /* 104 */ uint64_t blocksPerGroupBlockBitmap; // The number of blocks used to a store a group's block bitmap. /* 112 */ uint64_t gdtFirstBlock; // The first block in the group descriptor table. diff --git a/start.sh b/start.sh index fbd1405..bc09639 100755 --- a/start.sh +++ b/start.sh @@ -55,6 +55,6 @@ if [ ! -f "bin/good_compiler.txt" ]; then fi # Compile and run Build. -gcc -o bin/build -g util/build.c -Wall -Wextra -Wno-format-security -Wno-format-overflow \ - -Wno-missing-field-initializers -Wno-unused-function -Wno-format-truncation -pthread -DPARALLEL_BUILD \ +gcc -o bin/build -g util/build.c -pthread -DPARALLEL_BUILD -D${ES_TARGET-TARGET_X86_64} \ + -Wall -Wextra -Wno-format-security -Wno-format-overflow -Wno-missing-field-initializers -Wno-unused-function -Wno-format-truncation \ && bin/build "$@" diff --git a/util/build.c b/util/build.c index 791106a..41cc278 100644 --- a/util/build.c +++ b/util/build.c @@ -2,23 +2,20 @@ #define _GNU_SOURCE #endif -#if 0 +#if defined(TARGET_X86_64) #define TOOLCHAIN_PREFIX "x86_64-essence" #define TARGET_NAME "x86_64" #define TOOLCHAIN_HAS_RED_ZONE #define TOOLCHAIN_HAS_CSTDLIB #define QEMU_EXECUTABLE "qemu-system-x86_64" -#else +#elif defined(TARGET_X86_32) #define TOOLCHAIN_PREFIX "i686-elf" #define TARGET_NAME "x86_32" #define QEMU_EXECUTABLE "qemu-system-i386" +#else +#error Unknown target. #endif -#define WARNING_FLAGS \ - " -Wall -Wextra -Wno-missing-field-initializers -Wno-pmf-conversions -Wno-frame-address -Wno-unused-function -Wno-format-truncation -Wno-invalid-offsetof " -#define WARNING_FLAGS_C \ - " -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-function -Wno-format-truncation -Wno-unused-parameter " - #include #include @@ -355,11 +352,13 @@ void Compile(uint32_t flags, int partitionSize, const char *volumeLabel) { } void BuildUtilities() { +#define WARNING_FLAGS " -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-function -Wno-format-truncation -Wno-unused-parameter " + buildStartTimeStamp = time(NULL); #define BUILD_UTILITY(x, y, z) \ if (CheckDependencies("Utilities." x)) { \ - if (!CallSystem("gcc -MMD util/" z x ".c -o bin/" x " -g -std=c2x " WARNING_FLAGS_C " " y)) { \ + if (!CallSystem("gcc -MMD util/" z x ".c -o bin/" x " -g -std=c2x " WARNING_FLAGS " " y)) { \ ParseDependencies("bin/" x ".d", "Utilities." x, false); \ } \ } @@ -1124,6 +1123,8 @@ void DoCommand(const char *l) { BuildAndRun(OPTIMISE_ON, true /* compile */, false /* debug */, -1, LOG_NORMAL); } else if (0 == strcmp(l, "d") || 0 == strcmp(l, "debug")) { BuildAndRun(OPTIMISE_OFF, true /* compile */, true /* debug */, EMULATOR_QEMU, LOG_NORMAL); + } else if (0 == strcmp(l, "dlv")) { + BuildAndRun(OPTIMISE_OFF, true /* compile */, true /* debug */, EMULATOR_QEMU, LOG_VERBOSE); } else if (0 == strcmp(l, "d3") || 0 == strcmp(l, "debug-without-compile")) { BuildAndRun(OPTIMISE_OFF, false /* compile */, true /* debug */, EMULATOR_QEMU, LOG_NORMAL); } else if (0 == strcmp(l, "v") || 0 == strcmp(l, "vbox")) { @@ -1550,7 +1551,7 @@ int main(int _argc, char **_argv) { coloredOutput = isatty(STDERR_FILENO); if (argc == 1) { - printf(ColorHighlight "Essence Build" ColorNormal "\nPress Ctrl-C to exit.\n"); + printf(ColorHighlight "Essence Build" ColorNormal "\nPress Ctrl-C to exit.\nCross target is " ColorHighlight TARGET_NAME ColorNormal ".\n"); } systemLog = fopen("bin/system.log", "w"); diff --git a/util/build_core.c b/util/build_core.c index b3335c7..85347cb 100644 --- a/util/build_core.c +++ b/util/build_core.c @@ -1349,6 +1349,7 @@ int main(int argc, char **argv) { bootUseVBE = !!atoi(s.value); } else if (0 == strcmp(s.key, "Flag.COM_OUTPUT") && atoi(s.value)) { strcat(commonAssemblyFlags, " -DCOM_OUTPUT "); + strcat(commonCompileFlags, " -DCOM_OUTPUT "); } else if (0 == strcmp(s.key, "BuildCore.NoImportPOSIX")) { noImportPOSIX = !!atoi(s.value); } else if (0 == memcmp(s.key, "General.", 8)) {