Improve memory allocation layout

.gitignore

@@ -3,4 +3,6 @@
 local_notes/
 build/
 src_test/
-build_test/
+build_test/
+__pycache__/
+.vscode/

Makefile

@@ -55,8 +55,6 @@ MINIMAL_DISK = $(BUILD_DIR)/minimal_disk
 SRC_APP = $(SRC_DIR)/usr/local/src
 BUILD_APP = $(BUILD_DIR)/usr/local/bin
 
-MEMORY_LOCATION_KERNEL = 0xC000
-
 SOURCE_SNAPSHOT="\"$$(git rev-parse --short HEAD)$$(git diff --quiet || echo '_unstaged')\""
 
 # General Assumptions
@@ -77,7 +75,7 @@ endif
 # Tools
 HOST_CC = gcc -std=c11 -Iinclude
 
-NASM=nasm -f elf32 $(NASM_DEBUG)
+NASM=nasm -f elf32 -i include/ $(NASM_DEBUG)
 
 CC=gcc -std=c11 -fno-builtin -Os -nostartfiles -nostdlib -static $(CC_DEBUG)
 KERNEL_CC = $(CC) -m32 -fno-pie -Isrc --sysroot=$(BUILD_DIR) -Iinclude -Isrc/usr/include

before_install.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 apt-get update
 apt install -y make nasm gcc cpulimit qemu-system-x86 gocr expect gdb graphicsmagick-imagemagick-compat
+
+python3 -m pip install -r requirements.txt

emulator/Makefile.mk

@@ -1,6 +1,7 @@
 QEMU_SHUT_FLAGS= -no-shutdown -no-reboot
 QEMU_EXTRA_FLAGS=
 QEMU_GDB_PORT=9000
+QEMU_MONITOR_PORT=55555
 GDB_EX='echo Use GDB_EX="<command>" to execute command on connect.\n'
 
 qemu: $(image_vmdk)
@@ -9,6 +10,9 @@ qemu: $(image_vmdk)
 qemu_vvv: $(image_vmdk)
 	qemu-system-x86_64 -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) $(QEMU_EXTRA_FLAGS) -d  cpu,exec,in_asm
 
+qemu_monitor: $(image_vmdk)
+	qemu-system-x86_64 -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) $(QEMU_EXTRA_FLAGS) -monitor telnet:127.0.0.1:$(QEMU_MONITOR_PORT),server,nowait
+
 qemu_debug: $(image_vmdk)
 	qemu-system-x86_64 -S -gdb tcp::$(QEMU_GDB_PORT) -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) -d  cpu,exec,in_asm
 

include/fuzzy/kernel/interrupts/timer.h

@@ -4,4 +4,4 @@ void interrupt_pit_enable();
 
 void interrupt_register_0x20_irq0_pit();
 
-void create_infant_process_irq0_stack(int ds_ss_es_fs);
+int create_infant_process_irq0_stack(int ds_ss_es_fs);

include/fuzzy/memmgr/layout.asm

@@ -0,0 +1,17 @@
+; constants only
+
+; START_ENSURE_SAME_layout_h
+MEMORY_APP_SIZE EQU 0x20000
+
+; END_ENSURE_SAME_layout_h
+
+; Kernel core is app with pid 0
+; Kernel event is trigged via IRQ0 or syscall,
+STACKINIT_APP EQU (MEMORY_APP_SIZE-4)
+STACKINIT_KERNEL_CORE EQU (STACKINIT_APP-4)
+
+; Should be able to store
+; - syscall and all internal calls
+; - ISRs and all internal calls
+STACKINIT_KERNEL_EVENT EQU 0x18000
+

include/fuzzy/memmgr/layout.h

@@ -0,0 +1,17 @@
+#pragma once
+
+// START_ENSURE_SAME_layout_asm
+#define MEMORY_APP_SIZE         0x20000
+// END_ENSURE_SAME_layout_asm
+
+// Most of the memory layout relies on the fact the kernel is first app
+// and kernel's pid is 0.
+#define PID_KERNEL 0
+
+#define MEMORY_APPBASE_LOCATION 0x10000
+
+#define memmgr_app_abs_location(pid) (MEMORY_APPBASE_LOCATION + (pid)*MEMORY_APP_SIZE)
+#define memmgr_app_size(pid) (MEMORY_APP_SIZE)
+
+#define MEMORY_KERNEL_LOCATION memmgr_app_abs_location(PID_KERNEL)
+#define MEMORY_KERNEL_SIZE memmgr_app_size(PID_KERNEL)

requirements.txt

@@ -0,0 +1 @@
+pexpect

src/bootloader/Makefile.mk

@@ -28,7 +28,6 @@ $(bt_stage2).elf: $(SRC_BOOTLOADER)/stage2.asm $(SRC_BOOTLOADER)/stage2.c $(INCL
 		-D SECTOR_START_KERNEL=$(SECTOR_START_KERNEL) \
 		-D SECTOR_COUNT_KERNEL=$(SECTOR_COUNT_KERNEL) \
 		-D MEMORY_STATIC_LIBRARY=$(MEMORY_STATIC_LIBRARY) \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
 		-o $(BUILD_BOOTLOADER)/stage2_c.o $(SRC_BOOTLOADER)/stage2.c
 	$(LD) -m elf_i386 -Ttext 0x8000 -T linker.ld -o $@ $(BUILD_BOOTLOADER)/stage2_asm.o $(BUILD_BOOTLOADER)/stage2_c.o  $(BUILD_LIB_UTILS)/libutils_16 $(BUILD_DRIVERS)/display/libtm_bios $(BUILD_DRIVERS)/disk/libdisk_16
 	truncate --size=%512 $@

src/bootloader/stage2.c

@@ -1,3 +1,5 @@
+#include <fuzzy/memmgr/layout.h>
+
 #include <lib/utils/color.h>
 #include <drivers/disk/disk.h>
 #include <lib/utils/output.h>
@@ -29,12 +31,12 @@ char *get_memdump_8byte(void *address) {
 
 void load_kernel() {
     print_log("Loading Kernel");
-    int err = load_sectors(MEMORY_LOCATION_KERNEL, 0x80, SECTOR_START_KERNEL, SECTOR_COUNT_KERNEL);
+    int err = load_sectors(MEMORY_KERNEL_LOCATION, 0x80, SECTOR_START_KERNEL, SECTOR_COUNT_KERNEL);
     if(err) {
         print_log("Failed to load kernel in memory: %d", err);
         label_exit();
     } else {
-        print_log("Kernel loaded at 0x%x: %s...", MEMORY_LOCATION_KERNEL, get_memdump_8byte(MEMORY_LOCATION_KERNEL));
+        print_log("Kernel loaded at 0x%x: %s...", MEMORY_KERNEL_LOCATION, get_memdump_8byte(MEMORY_KERNEL_LOCATION));
     }
 }
 

src/drivers/disk/disk.c

@@ -1,3 +1,4 @@
+// written for protected mode
 #include <lib/utils/logging.h>
 #include <fuzzy/real_mode/client.h>
 

src/drivers/disk/disk.h

@@ -1,6 +1,6 @@
 #pragma once
 
-int load_sectors(unsigned int address,
+int load_sectors(unsigned int full_address,
                  unsigned char drive,
-                 unsigned int sector_index, // 1-based
+                 unsigned int lba, // 0-based
                  unsigned char count);

src/drivers/disk/disk_16.asm

@@ -1,29 +1,28 @@
 [BITS 16]
 
-global _low_disk_read_sectors
+global _low_int_0x13
 
 [SECTION .text]
-    _low_disk_read_sectors:
+    _low_int_0x13:
         push ebp
         mov ebp, esp
         ; callee save register
         push ebx
         push esi
         push edi
+        push es
 
         ; check es
-        mov ah, 0x02                ; (read sectors)
-        mov al, [ebp + 0x8]         ; (sector count)
-        mov cx, [ebp + 0xc]         ; (cylinder 10bit, sector 6bit)
-        mov dx, [ebp + 0x10]        ; (head, drive index)
-        mov bx, [ebp + 0x14]        ; (es:bx as write address)
-        int 0x13
+        mov ax, [ebp + 0x18]        ; (es)
+        mov es, ax
 
-        mov ah, 0x01                 ; (get status of last drive operation)
+        mov ax, [ebp + 0x08]
+        mov bx, [ebp + 0x0C]
+        mov cx, [ebp + 0x10]
+        mov dx, [ebp + 0x14]
         int 0x13
-        mov al, ah                   ; (result status code, 0 means no error)
-        and eax, 0xFF
 
+        pop es
         pop edi
         pop esi
         pop ebx

src/drivers/disk/disk_16.c

@@ -1,18 +1,35 @@
-extern int _low_disk_read_sectors(
-    unsigned char count,
-    unsigned short cylinder_sector,  // 10 bit + 6bit
-    unsigned short head_driveindex, // 8 bit + 8bit
-    unsigned short memory_address);
+// written for real mode
 
-int load_sectors(unsigned int address,
+extern unsigned short _low_int_0x13(unsigned short ax,
+                                    unsigned short bx,
+                                    unsigned short cx,
+                                    unsigned short dx,
+                                    unsigned short es);
+
+int load_sectors(unsigned int full_address,
                  unsigned char drive,
-                 unsigned int sector_index, // 0-based
+                 unsigned int lba, // 0-based
                  unsigned char count) {
-    int cylinder_head = (sector_index/63);
-    sector_index = sector_index%63 + 1;
-    _low_disk_read_sectors(
-        count,
+    int es = (full_address&0xF0000)>>4;
+    int es_address = full_address&0xFFFF;
+    int cylinder_head = (lba/63);
+    int sector_index = lba%63 + 1;
+    // https://en.wikipedia.org/wiki/INT_13H#INT_13h_AH=02h:_Read_Sectors_From_Drive
+    _low_int_0x13(
+        (0x02<<8) | count,
+        es_address,
         ((cylinder_head>>2)&0xFFC0) | (sector_index),
         ((cylinder_head<<8)&0xFF00) | (drive),
-        address);
-}
+        es
+    );
+    // https://en.wikipedia.org/wiki/INT_13H#INT_13h_AH=01h:_Get_Status_of_Last_Drive_Operation
+    unsigned short ax = _low_int_0x13(
+        (0x01<<8),
+        0,
+        0,
+        drive,
+        0
+    );
+    int status = ax >> 8;
+    return status;
+}

src/drivers/keyboard/Makefile.mk

@@ -2,7 +2,6 @@
 $(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(BUILD_USR_INCLUDE_ALL)
 	mkdir -p $(dir $@)
 	$(KERNEL_CC) -c -o $@ \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
 		$<
 
 $(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm

src/fs/Makefile.mk

@@ -1,7 +1,6 @@
 $(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(SELF_INCLUDE_DIR)/%.h
 	mkdir -p $(dir $@)
 	$(KERNEL_CC) -c \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
 		-o $@ $<
 
 $(SELF_BUILD_DIR)/libffs:	$(patsubst $(SELF_SRC_DIR)/%.c,$(SELF_BUILD_DIR)/%.o,$(wildcard $(SELF_SRC_DIR)/*.c))

src/fs/ffs.c

@@ -1,5 +1,6 @@
 #include <fuzzy/fs/ffs.h>
 #include <fuzzy/fs/mbr.h>
+#include <fuzzy/memmgr/layout.h>
 #include <string.h>
 #include <drivers/disk/disk.h>
 #include <lib/utils/logging.h>
@@ -8,7 +9,7 @@
 // TODO: verify partition metadata
 
 int partition_read_block(int lba, void *wr_buffer) {
-    int memory_location = ((int)wr_buffer)+MEMORY_LOCATION_KERNEL;
+    int memory_location = ((int)wr_buffer)+MEMORY_KERNEL_LOCATION;
     int err = load_sectors(memory_location, 0x80, lba, 1);
     return err;
 }

src/fs/mbr.c

@@ -1,15 +1,17 @@
 #include <fuzzy/fs/ffs.h>
 #include <fuzzy/fs/mbr.h>
+#include <fuzzy/kernel/panic.h>
+#include <fuzzy/memmgr/layout.h>
+
 #include <drivers/disk/disk.h>
 #include <lib/utils/logging.h>
-#include <fuzzy/kernel/panic.h>
 
 char _cache_mbrblock[FS_BLOCK_SIZE];
 int _cache_mbrblock_ready = 0;
 
 char *get_mbrblock() {
     if(!_cache_mbrblock_ready) {
-        int memory_location = ((int)_cache_mbrblock)+MEMORY_LOCATION_KERNEL;
+        int memory_location = ((int)_cache_mbrblock)+MEMORY_KERNEL_LOCATION;
         int lba = 0;
         int err = load_sectors(memory_location, 0x80, lba, 1);
         if (err) {

src/kernel/Makefile.mk

@@ -1,5 +1,3 @@
-MEMORY_LOCATION_APP = 0x20000
-
 debug_kernel: $(kernel_core)
 	objdump -b binary -mi386 -Maddr32,data32 -D $<
 	xxd $<
@@ -9,8 +7,6 @@ $(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(BUILD_USR_INCLUDE_ALL)
 	$(KERNEL_CC) -c -o $@ \
 		-D INIT_APPNAME=\"$(INIT_APPNAME)\" \
 		-D __SOURCE_SNAPSHOT__=$(SOURCE_SNAPSHOT) \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
-		-D MEMORY_LOCATION_APP=$(MEMORY_LOCATION_APP) \
 		$<
 
 $(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm

src/kernel/core.asm

@@ -1,3 +1,5 @@
+%include "fuzzy/memmgr/layout.asm"
+
 [BITS 32]
 
 extern kernel_core_entry
@@ -14,9 +16,7 @@ global kernel_core_entry_asm
         mov fs, ax
         mov gs, ax
 
-        ; TODO: Increase kernel code stack memory size
-        ; The value also depends on kernel handler stack size
-        mov esp, 0xC000  ; init stack pointer
+        mov esp, STACKINIT_KERNEL_CORE  ; init stack pointer
         jmp kernel_core_entry
 
         ; kernel_core_entry_asm currently exists only for tests.

src/kernel/interrupts/Makefile.mk

@@ -2,7 +2,6 @@
 $(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(BUILD_USR_INCLUDE_ALL)
 	mkdir -p $(dir $@)
 	$(KERNEL_CC) -c -o $@ \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
 		$<
 
 $(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm

src/kernel/interrupts/interrupts.c

@@ -1,6 +1,7 @@
 #include <fuzzy/kernel/syscall/syscall.h>
 #include <fuzzy/kernel/interrupts/interrupts.h>
 #include <fuzzy/kernel/interrupts/timer.h>
+#include <fuzzy/memmgr/layout.h>
 
 #include <lib/utils/logging.h>
 
@@ -71,7 +72,7 @@ void populate_and_load_idt_table() {
     interrupt_register_0x32_syscall();
 
     idtr.size = sizeof(struct IDTEntry)*IDT_SIZE;
-    idtr.base_address = ((int)idt_table + MEMORY_LOCATION_KERNEL);
+    idtr.base_address = ((int)idt_table + MEMORY_KERNEL_LOCATION);
     print_log("IDTR: 0x%x; base address: 0x%x, size: %d",
         (int)&idtr, idtr.base_address, idtr.size);
     reload_idt_table();

src/kernel/interrupts/timer.asm

@@ -1,3 +1,5 @@
+%include "fuzzy/memmgr/layout.asm"
+
 [BITS 32]
 
 global irq0_pit_handler_low
@@ -25,7 +27,7 @@ global create_infant_process_irq0_stack
         mov esi, esp
 
         ; new stack and segment area
-        mov esp, 0xFFF0  ; keep offset in sync with process_create(...) and create_infant_process_irq0_stack
+        mov esp, STACKINIT_KERNEL_EVENT
         mov eax, 0x10
         mov ss, eax
         mov ds, eax
@@ -82,6 +84,8 @@ global create_infant_process_irq0_stack
         iret
 
     create_infant_process_irq0_stack:
+        ; keep in sync with _int_irq0_start
+        ; return the esp for user stack
         push ebp
         mov ebp, esp
 
@@ -95,9 +99,8 @@ global create_infant_process_irq0_stack
 
         mov ds, ecx
 
-        ; user stack creation start
         ; user initial stack
-        mov eax, 0xFFF0 ; keep in sync with _int_irq0_start
+        mov eax, STACKINIT_APP
         ; kernel offset
         xor ecx, ecx
         mov [eax-0], ecx    ; user: eflag
@@ -116,9 +119,7 @@ global create_infant_process_irq0_stack
         mov [eax-52], ecx       ; user: fs
         mov [eax-56], ecx       ; user: gs
 
-        ; user: esp = eax-56 = 0xFFF0-56
-        ; should be compatible with process_create
-        ; user stack creation end
+        sub eax, 56 ; user stack pointer
 
         pop ds
 

src/kernel/process/Makefile.mk

@@ -1,8 +1,6 @@
 $(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(BUILD_USR_INCLUDE_ALL)
 	mkdir -p $(dir $@)
 	$(KERNEL_CC) -c -o $@ \
-		-D MEMORY_LOCATION_KERNEL=$(MEMORY_LOCATION_KERNEL) \
-		-D MEMORY_LOCATION_APP=$(MEMORY_LOCATION_APP) \
 		$<
 
 $(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm