Last stage of the memory system rewrite. This adds the last support f…

…or memory buffers, also fixed issues with file mappings and added better support for allocations and custom mappings. Everything now needs to be tested.

.vscode/c_cpp_properties.json

@@ -77,6 +77,7 @@
                 "__amd64__",
                 "UNICODE",
                 "_USE_64BIT_TIME_T",
+                "_INTEGRAL_MAX_BITS=64",
                 "__need_size_t",
                 "__INT_MAX__=2147483647",
                 "__INT64_TYPE__=long long",

.vscode/settings.json

@@ -321,7 +321,8 @@
         "atomicsection.h": "c",
         "semaphore_slim.h": "c",
         "memoryspace.h": "c",
-        "machine.h": "c"
+        "machine.h": "c",
+        "simd": "cpp"
     },
     "git.ignoreLimitWarning": true,
     "terminal.integrated.shell.windows": "C:\\Windows\\sysnative\\bash.exe"

kernel/acpi/hpet.c

@@ -392,7 +392,7 @@ HpInitialize(
     // Map the address
     Status = CreateSystemMemorySpaceMapping(GetCurrentSystemMemorySpace(), 
         &HpetController.BaseAddress, &HpetController.BaseAddress, GetSystemMemoryPageSize(),
-        MAPPING_VIRTUAL | MAPPING_NOCACHE | MAPPING_DMA, __MASK);
+        MAPPING_PROVIDED | MAPPING_PERSISTENT | MAPPING_NOCACHE | MAPPING_KERNEL, __MASK);
     if (Status != OsSuccess) {
         ERROR("Failed to map address for hpet.");
         return OsError;

kernel/arch/x86/components/memory.c

@@ -30,9 +30,9 @@
 #include <apic.h>
 
 // Global static storage for the memory
-static _Atomic(uintptr_t) ReservedMemoryPointer = ATOMIC_VAR_INIT(MEMORY_LOCATION_RESERVED);
-static MemorySynchronizationObject_t SyncData   = { SPINLOCK_INIT, 0 };
-static size_t BlockmapBytes                     = 0;
+static BlockBitmap_t                    KernelMemory    = { 0 };
+static MemorySynchronizationObject_t    SyncData        = { SPINLOCK_INIT, 0 };
+static size_t                           BlockmapBytes   = 0;
 extern void memory_invalidate_addr(uintptr_t pda);
 
 /* MmMemoryDebugPrint
@@ -59,6 +59,7 @@ InitializeSystemMemory(
     // Variables
     BIOSMemoryRegion_t *RegionPointer = NULL;
     uintptr_t MemorySize;
+    size_t BytesOccupied = 0;
     int i;
 
     // Initialize
@@ -72,6 +73,7 @@ InitializeSystemMemory(
     ConstructBlockmap(Memory, (void*)MEMORY_LOCATION_BITMAP, 
         BLOCKMAP_ALLRESERVED, 0, MemorySize, PAGE_SIZE);
     BlockmapBytes = GetBytesNeccessaryForBlockmap(0, MemorySize, PAGE_SIZE);
+    BytesOccupied += BlockmapBytes + PAGE_SIZE;
 
     // Free regions given to us by memory map
     for (i = 0; i < (int)BootInformation->MemoryMapLength; i++) {
@@ -82,6 +84,11 @@ InitializeSystemMemory(
         RegionPointer++;
     }
 
+    // Initialize the kernel memory region
+    ConstructBlockmap(&KernelMemory, (void*)(MEMORY_LOCATION_BITMAP + (BlockmapBytes + PAGE_SIZE)), 
+        0, MEMORY_LOCATION_RESERVED, MEMORY_LOCATION_KERNEL_END, PAGE_SIZE);
+    BytesOccupied += GetBytesNeccessaryForBlockmap(MEMORY_LOCATION_RESERVED, MEMORY_LOCATION_KERNEL_END, PAGE_SIZE) + PAGE_SIZE;
+
     // Mark default regions in use and special regions
     //  0x0000              || Used for catching null-pointers
     //  0x4000 + 0x8000     || Used for memory region & Trampoline-code
@@ -93,7 +100,7 @@ InitializeSystemMemory(
     ReserveBlockmapRegion(Memory, 0x90000,                  0xF000);
     ReserveBlockmapRegion(Memory, MEMORY_LOCATION_KERNEL,   BootInformation->KernelSize + PAGE_SIZE);
     ReserveBlockmapRegion(Memory, MEMORY_LOCATION_RAMDISK,  BootInformation->RamdiskSize + PAGE_SIZE);
-    ReserveBlockmapRegion(Memory, MEMORY_LOCATION_BITMAP,   (BlockmapBytes + PAGE_SIZE));
+    ReserveBlockmapRegion(Memory, MEMORY_LOCATION_BITMAP,   BytesOccupied);
 
     // Fill in rest of data
     *MemoryGranularity      = PAGE_SIZE;
@@ -133,8 +140,8 @@ ConvertSystemSpaceToPaging(Flags_t Flags)
 	if (Flags & MAPPING_NOCACHE) {
 		NativeFlags |= PAGE_CACHE_DISABLE;
 	}
-	if (Flags & MAPPING_VIRTUAL) {
-		NativeFlags |= PAGE_VIRTUAL;
+	if (Flags & MAPPING_PERSISTENT) {
+		NativeFlags |= PAGE_PERSISTENT;
 	}
     if (!(Flags & MAPPING_READONLY)) {
         NativeFlags |= PAGE_WRITE;
@@ -165,26 +172,15 @@ ConvertPagingToSystemSpace(Flags_t Flags)
     if (Flags & PAGE_CACHE_DISABLE) {
         GenericFlags |= MAPPING_NOCACHE;
     }
-    if (Flags & PAGE_VIRTUAL) {
-        GenericFlags |= MAPPING_VIRTUAL;
+    if (Flags & PAGE_PERSISTENT) {
+        GenericFlags |= MAPPING_PERSISTENT;
     }
     if (Flags & PAGE_DIRTY) {
         GenericFlags |= MAPPING_ISDIRTY;
     }
     return GenericFlags;
 }
 
-/* MmReserveMemory
- * Reserves memory for system use - should be allocated
- * from a fixed memory region that won't interfere with
- * general usage */
-VirtualAddress_t*
-MmReserveMemory(
-	_In_ int Pages)
-{
-	return (VirtualAddress_t*)atomic_fetch_add(&ReservedMemoryPointer, (PAGE_SIZE * Pages));
-}
-
 /* PageSynchronizationHandler
  * Synchronizes the page address specified in the MemorySynchronization Object. */
 InterruptStatus_t
@@ -260,8 +256,8 @@ ResolveVirtualSpaceAddress(
     _CRT_UNUSED(SystemMemorySpace);
     _CRT_UNUSED(PhysicalBase);
 
-    if (Flags & MAPPING_DMA) {
-        *VirtualBase = (VirtualAddress_t)MmReserveMemory(DIVUP(Size, PAGE_SIZE));
+    if (Flags & MAPPING_KERNEL) {
+        *VirtualBase = AllocateBlocksInBlockmap(&KernelMemory, __MASK, Size);
         return OsSuccess;
     }
     else if (Flags & MAPPING_LEGACY) {
@@ -270,3 +266,13 @@ ResolveVirtualSpaceAddress(
     }
     return OsError;
 }
+
+/* ClearKernelMemoryAllocation
+ * Clears the kernel memory allocation at the given address and size. */
+OsStatus_t
+ClearKernelMemoryAllocation(
+    _In_ uintptr_t              Address,
+    _In_ size_t                 Size)
+{
+    return ReleaseBlockmapRegion(&KernelMemory, Address, Size);
+}

kernel/arch/x86/components/smp_trampoline.c

@@ -1,7 +1,7 @@
 /* Automatically generated file, do not change contents.
  * The below file data is from build/ap.sys. */
 
-const char __GlbTramplineCode[677] = {
+const char __GlbTramplineCode[618] = {
     0xe9, 0x09, 0x01, 0x9c, 0x1e, 0x06, 0x57, 0x56, 0xfa, 0x31, 0xc0, 
     0x8e, 0xc0, 0xf7, 0xd0, 0x8e, 0xd8, 0xbf, 0x00, 0x05, 0xbe, 0x10, 
     0x05, 0x26, 0x8a, 0x05, 0x50, 0x3e, 0x8a, 0x04, 0x50, 0x26, 0xc6, 
@@ -28,7 +28,7 @@ const char __GlbTramplineCode[677] = {
     0x00, 0xbe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 
     0x00, 0x00, 0x00, 0x00, 0xfa, 0xea, 0x12, 0x81, 0x00, 0x00, 0x31, 
     0xc0, 0x8e, 0xd8, 0x8e, 0xc0, 0x8e, 0xe0, 0x8e, 0xe8, 0x8e, 0xd0, 
-    0xa1, 0x99, 0x82, 0xf0, 0x0f, 0xc1, 0x06, 0x9b, 0x82, 0x89, 0xc4, 
+    0xa1, 0x5e, 0x82, 0xf0, 0x0f, 0xc1, 0x06, 0x60, 0x82, 0x89, 0xc4, 
     0x89, 0xc5, 0x31, 0xc0, 0xfb, 0xe8, 0x56, 0xff, 0xe8, 0x7e, 0xff, 
     0x0f, 0x20, 0xc0, 0x66, 0x83, 0xc8, 0x01, 0x0f, 0x22, 0xc0, 0xea, 
     0xfb, 0x81, 0x08, 0x00, 0x60, 0xb8, 0x01, 0x00, 0x00, 0x80, 0x0f, 
@@ -49,21 +49,16 @@ const char __GlbTramplineCode[677] = {
     0xc0, 0x0f, 0x32, 0x0d, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x30, 0x0f, 
     0x20, 0xc0, 0x0d, 0x00, 0x00, 0x00, 0x80, 0x0f, 0x22, 0xc0, 0x61, 
     0xc3, 0xfa, 0x31, 0xc0, 0x66, 0xb8, 0x10, 0x00, 0x8e, 0xd8, 0x8e, 
-    0xe0, 0x8e, 0xe8, 0x8e, 0xd0, 0x8e, 0xc0, 0x0f, 0xb7, 0xe5, 0xe8, 
-    0x2f, 0xff, 0xff, 0xff, 0x83, 0xf8, 0x01, 0x75, 0x31, 0xa1, 0x9d, 
-    0x82, 0x00, 0x00, 0x0f, 0x22, 0xd8, 0x0f, 0x20, 0xe0, 0x83, 0xc8, 
-    0x20, 0x0f, 0x22, 0xe0, 0xb9, 0x80, 0x00, 0x00, 0xc0, 0x0f, 0x32, 
-    0x0d, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x30, 0x0f, 0x20, 0xc0, 0x0d, 
-    0x00, 0x00, 0x00, 0x80, 0x0f, 0x22, 0xc0, 0xea, 0x6b, 0x82, 0x00, 
-    0x00, 0x28, 0x00, 0xa1, 0x9d, 0x82, 0x00, 0x00, 0x0f, 0x22, 0xd8, 
-    0x0f, 0x20, 0xc0, 0x0d, 0x00, 0x00, 0x00, 0x80, 0x0f, 0x22, 0xc0, 
-    0x31, 0xf6, 0x31, 0xff, 0x8b, 0x0d, 0xa1, 0x82, 0x00, 0x00, 0xff, 
-    0xe1, 0xfa, 0xf4, 0x31, 0xc0, 0x66, 0xb8, 0x30, 0x00, 0x8e, 0xd8, 
-    0x8e, 0xe0, 0x8e, 0xe8, 0x8e, 0xd0, 0x8e, 0xc0, 0x48, 0x0f, 0xb7, 
-    0xe5, 0x48, 0x31, 0xf6, 0x48, 0x31, 0xff, 0x48, 0x31, 0xc0, 0x48, 
-    0x31, 0xdb, 0x48, 0x31, 0xc9, 0x8b, 0x0c, 0x25, 0xa1, 0x82, 0x00, 
-    0x00, 0xff, 0xe1, 0xfa, 0xf4, 0x00, 0x05, 0x00, 0x15, 0x00, 0x00, 
-    0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+    0xe0, 0x8e, 0xe8, 0x8e, 0xd0, 0x8e, 0xc0, 0x0f, 0xb7, 0xe5, 0xa1, 
+    0x62, 0x82, 0x00, 0x00, 0x0f, 0x22, 0xd8, 0x0f, 0x20, 0xc0, 0x0d, 
+    0x00, 0x00, 0x00, 0x80, 0x0f, 0x22, 0xc0, 0x31, 0xf6, 0x31, 0xff, 
+    0x8b, 0x0d, 0x66, 0x82, 0x00, 0x00, 0xff, 0xe1, 0xfa, 0xf4, 0x31, 
+    0xc0, 0x66, 0xb8, 0x30, 0x00, 0x8e, 0xd8, 0x8e, 0xe0, 0x8e, 0xe8, 
+    0x8e, 0xd0, 0x8e, 0xc0, 0x48, 0x0f, 0xb7, 0xe5, 0x48, 0x31, 0xf6, 
+    0x48, 0x31, 0xff, 0x48, 0x31, 0xc0, 0x48, 0x31, 0xdb, 0x48, 0x31, 
+    0xc9, 0x8b, 0x0c, 0x25, 0x66, 0x82, 0x00, 0x00, 0xff, 0xe1, 0xfa, 
+    0xf4, 0x00, 0x05, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+    0x00, 0x00
 };
 
-const int __GlbTramplineCode_length = 677;
+const int __GlbTramplineCode_length = 618;

kernel/arch/x86/interrupts/apic/apicinit.c

@@ -151,7 +151,7 @@ ParseIoApic(
     Original = Controller->MemoryAddress;
     CreateSystemMemorySpaceMapping(GetCurrentSystemMemorySpace(),
         &Original, &Updated, GetSystemMemoryPageSize(), 
-        MAPPING_NOCACHE | MAPPING_DMA | MAPPING_VIRTUAL, __MASK);
+        MAPPING_NOCACHE | MAPPING_KERNEL | MAPPING_PERSISTENT | MAPPING_PROVIDED, __MASK);
     Controller->MemoryAddress = Updated + (Original & 0xFFF);
 
     /* Maximum Redirection Entry - RO. This field contains the entry number (0 being the lowest
@@ -443,7 +443,7 @@ ApicInitialize(void)
     TRACE(" > local apic at 0x%x", OriginalApAddress);
     CreateSystemMemorySpaceMapping(GetCurrentSystemMemorySpace(),
         &OriginalApAddress, &UpdatedApAddress, GetSystemMemoryPageSize(), 
-        MAPPING_NOCACHE | MAPPING_DMA | MAPPING_VIRTUAL, __MASK);
+        MAPPING_NOCACHE | MAPPING_KERNEL | MAPPING_PERSISTENT | MAPPING_PROVIDED, __MASK);
     GlbLocalApicBase    = UpdatedApAddress + (OriginalApAddress & 0xFFF);
     BspApicId           = (ApicReadLocal(APIC_PROCESSOR_ID) >> 24) & 0xFF;
 

kernel/arch/x86/memory.h

@@ -49,7 +49,7 @@
 /* MollenOS PT/Page Definitions */
 #define PAGE_SYSTEM_MAP         0x200
 #define PAGE_INHERITED          0x400
-#define PAGE_VIRTUAL            0x800
+#define PAGE_PERSISTENT         0x800
 
 /* Memory Map Structure 
  * This is the structure passed to us by the mBoot bootloader */
@@ -88,18 +88,17 @@ SynchronizeVirtualPage(
     _In_ SystemMemorySpace_t*   SystemMemorySpace,
     _In_ uintptr_t              Address);
 
-/* MmReserveMemory
- * Reserves memory for system use - should be allocated
- * from a fixed memory region that won't interfere with
- * general usage */
-KERNELAPI VirtualAddress_t* KERNELABI
-MmReserveMemory(
-    _In_ int                Pages);
-
 /* PageSynchronizationHandler
  * Synchronizes the page address specified in the MemorySynchronization Object. */
 KERNELAPI InterruptStatus_t KERNELABI
 PageSynchronizationHandler(
     _In_ void*              Context);
 
+/* ClearKernelMemoryAllocation
+ * Clears the kernel memory allocation at the given address and size. */
+KERNELAPI OsStatus_t KERNELABI
+ClearKernelMemoryAllocation(
+    _In_ uintptr_t              Address,
+    _In_ size_t                 Size);
+
 #endif // !_X86_MEMORY_H_

kernel/arch/x86/x32/memory/virtualmemory.c

@@ -252,6 +252,13 @@ SetVirtualPageAttributes(
         return OsError;
     }
 
+    // For kernel mappings we would like to mark the mappings global
+    if (Address < MEMORY_LOCATION_KERNEL_END) {
+        if (CpuHasFeatures(0, CPUID_FEAT_EDX_PGE) == OsSuccess) {
+            ConvertedFlags |= PAGE_GLOBAL;
+        }  
+    }
+
 	// Map it, make sure we mask the page address so we don't accidently set any flags
     Mapping = atomic_load(&Table->Pages[PAGE_TABLE_INDEX(Address)]);
     if (!(Mapping & PAGE_SYSTEM_MAP)) {
@@ -336,7 +343,7 @@ SetVirtualPageMapping(
     Mapping = atomic_load(&Table->Pages[PAGE_TABLE_INDEX((vAddress & PAGE_MASK))]);
 SyncTable:
     if (Mapping != 0) {
-        if (ConvertedFlags & PAGE_VIRTUAL) {
+        if (ConvertedFlags & PAGE_PERSISTENT) {
             if (Mapping != (pAddress & PAGE_MASK)) {
                 FATAL(FATAL_SCOPE_KERNEL, 
                     "Tried to remap fixed virtual address 0x%x => 0x%x (Existing 0x%x)", 
@@ -383,6 +390,12 @@ ClearVirtualPageMapping(
     if (Table == NULL) {
         return OsError;
     }
+
+    // For kernel mappings we would like to mark the page free if it's
+    // in the kernel reserved region
+    if (Address >= MEMORY_LOCATION_RESERVED && Address < MEMORY_LOCATION_KERNEL_END) {
+        ClearKernelMemoryAllocation(Address, PAGE_SIZE);
+    }
 
     // Load the mapping
     Mapping = atomic_load(&Table->Pages[PAGE_TABLE_INDEX(Address)]);
@@ -397,7 +410,7 @@ ClearVirtualPageMapping(
 
             // Release memory, but don't if it is a virtual mapping, that means we 
             // should not free the physical page
-            if (!(Mapping & PAGE_VIRTUAL)) {
+            if (!(Mapping & PAGE_PERSISTENT)) {
                 FreeSystemMemory(Mapping & PAGE_MASK, PAGE_SIZE);
             }
 
@@ -542,7 +555,7 @@ DestroyVirtualSpace(
         Table = (PageTable_t*)Pd->vTables[i];
         for (j = 0; j < ENTRIES_PER_PAGE; j++) {
             CurrentMapping = atomic_load_explicit(&Table->Pages[j], memory_order_relaxed);
-            if (CurrentMapping & PAGE_VIRTUAL) {
+            if (CurrentMapping & PAGE_PERSISTENT) {
                 continue;
             }