Create and add simple filesystem FFS

.gdbinit

@@ -0,0 +1,92 @@
+set $CS_BASE = 0x20000
+set $DS_BASE = 0x20000
+set $SS_BASE = 0x20000
+
+# config
+set architecture i386:x86-64
+set pagination off
+set prompt \033[32mgdb$ \033[0m
+
+# layout
+define cb
+    b *($CS_BASE+$arg0)
+end
+
+define cprintstr
+    x/s $DS_BASE+$arg0
+end
+
+define cprintdata
+    x/4w $DS_BASE+$arg0
+end
+
+define casm
+    x/20i ($CS_BASE+$pc)
+end
+
+define cstack
+    info register esp ebp
+    x/24w $SS_BASE+$esp
+end
+
+define creg
+    # Not needed for now: cr0 cr2 cr3 cr4 cr8 gs_base
+    info register eax ebx ecx edx esi edi eflags cs ss ds es fs gs fs_base
+end
+
+define _clayout_title
+    echo \033[33m---[ $arg0 ]---------------\033[0m\n
+end
+
+define clayout
+    # clear screen
+    echo \033c
+    _clayout_title ASM
+    casm
+    _clayout_title Register
+    creg
+    _clayout_title Stack
+    cstack
+    _clayout_title Command
+end
+
+define cc
+    # shorthand for cresetscreen
+    clayout
+end
+
+# mode
+define view_realmode
+    set $CS_BASE = 0x0000
+    set $DS_BASE = 0x0000
+    set $SS_BASE = 0x0000
+    clayout
+end
+
+define view_kernelmode
+    set $CS_BASE = 0xC000
+    set $DS_BASE = 0xC000
+    set $SS_BASE = 0xC000
+    clayout
+end
+
+define view_usermode
+    set $CS_BASE = 0x20000
+    set $DS_BASE = 0x20000
+    set $SS_BASE = 0x20000
+    clayout
+end
+
+# override existing shorthand
+define si
+    stepi
+    cc
+end
+define ni
+    nexti
+    cc
+end
+define c
+    continue
+    cc
+end

Makefile

@@ -1,11 +1,19 @@
+ROOT_DIR = .
 BUILD_DIR = build
+INCLUDE_DIR = include/fuzzy
 SRC_DIR = src
 
+SELF_SRC_DIR = $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_LIST))))
+SELF_INCLUDE_DIR = $(patsubst $(SRC_DIR)/%,$(INCLUDE_DIR)/%,$(SELF_SRC_DIR))
+SELF_BUILD_DIR = $(patsubst $(SRC_DIR)/%,$(BUILD_DIR)/%,$(SELF_SRC_DIR))
+
+SELF_SRC_ALL_C = $(shell find $(SELF_SRC_DIR) -name '*.c')
+SELF_BUILD_ALL_C = $(patsubst $(SELF_SRC_DIR)/%.c,$(SELF_BUILD_DIR)/%.o,$(SELF_SRC_ALL_C))
+
 SRC_BOOTLOADER = $(SRC_DIR)/bootloader
 SRC_KERNEL = $(SRC_DIR)/kernel
 SRC_DRIVERS = $(SRC_DIR)/drivers
 SRC_LIB_DS = $(SRC_DIR)/lib/ds
-SRC_LIB_SYSCALL = $(SRC_DIR)/lib/syscall
 SRC_LIB_UTILS = $(SRC_DIR)/lib/utils
 SRC_LIB = $(SRC_DIR)/lib
 SRC_MEMMGR = $(SRC_DIR)/memmgr
@@ -15,7 +23,6 @@ BUILD_BOOTLOADER = $(BUILD_DIR)/bootloader
 BUILD_KERNEL = $(BUILD_DIR)/kernel
 BUILD_DRIVERS = $(BUILD_DIR)/drivers
 BUILD_LIB_DS = $(BUILD_DIR)/lib/ds
-BUILD_LIB_SYSCALL = $(BUILD_DIR)/lib/syscall
 BUILD_LIB_UTILS = $(BUILD_DIR)/lib/utils
 BUILD_LIB = $(BUILD_DIR)/lib
 BUILD_REALMODE = $(BUILD_DIR)/real_mode
@@ -31,11 +38,20 @@ rm_static = $(BUILD_REALMODE)/static_library
 # Kernel
 kernel_core = $(BUILD_DIR)/kernel/core
 
+# Program to auto start when kernel is ready.
+# 1 - Tic Tac Toe
+# 2 - Calculator
+# 3 - ls
+# 4 - cat
+RUN_APP_ID = 1
+
 # Apps
 SRC_APP = $(SRC_DIR)/usr/local/src
 BUILD_APP = $(BUILD_DIR)/usr/local/bin
 app_calc = $(BUILD_APP)/calc.out
 app_tic_tac_toe = $(BUILD_APP)/tic_tac_toe.out
+app_ls = $(BUILD_APP)/ls.out
+app_cat = $(BUILD_APP)/cat.out
 
 MEMORY_LOCATION_KERNEL = 0xC000
 
@@ -46,16 +62,12 @@ SOURCE_SNAPSHOT="\"$$(git rev-parse --short HEAD)$$(git diff --quiet || echo '_u
 
 # Tools
 CC=gcc -std=c11 -fno-builtin -Os -nostartfiles -nostdlib -static
-KERNEL_CC = $(CC) -m32 -fno-pie -Isrc --sysroot=$(BUILD_DIR)
+HOST_CC = gcc -std=c11 -Iinclude
+KERNEL_CC = $(CC) -m32 -fno-pie -Isrc --sysroot=$(BUILD_DIR) -Iinclude -Isrc/usr/include
 LD=ld  -nostdlib -nostartfiles -nodefaultlibs --strip-all # --print-map
 
-# Program to auto start when kernel is ready.
-# 1 - Tic Tac Toe
-# 2 - Calculator (broken)
-RUN_APP_ID = 1
-
 # Targets
-all_artifacts: images binaries
+all_artifacts: images binaries external
 
 test: $(image_vmdk) $(wildcard tests/**/*)
 	bash tests/run.sh
@@ -66,34 +78,39 @@ images: $(image_vmdk)
 binaries: $(bt_stage1) $(bt_stage2) $(kernel_core) $(rm_static)
 
 # Build dependecies for configure and $(image_vmdk) are ordered based on their position on disk.
+# Note: Relying on default value doesn't guarantee bin size.
 SECTOR_START_BT_STAGE1 = 0
-SECTOR_COUNT_BT_STAGE1 = $(shell cut -d' ' -f1 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_BT_STAGE1 = $(shell cut -d' ' -f1 configure 2> /dev/null || echo 1 )
 SECTOR_START_SHARED_LIBRARY = $(shell expr $(SECTOR_START_BT_STAGE1) + $(SECTOR_COUNT_BT_STAGE1) )
-SECTOR_COUNT_SHARED_LIBRARY = $(shell cut -d' ' -f2 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_SHARED_LIBRARY = $(shell cut -d' ' -f2 configure 2> /dev/null || echo 1 )
 SECTOR_START_BT_STAGE2 = $(shell expr $(SECTOR_START_SHARED_LIBRARY) + $(SECTOR_COUNT_SHARED_LIBRARY) )
-SECTOR_COUNT_BT_STAGE2 = $(shell cut -d' ' -f3 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_BT_STAGE2 = $(shell cut -d' ' -f3 configure 2> /dev/null || echo 30 )
 SECTOR_START_KERNEL = $(shell expr $(SECTOR_START_BT_STAGE2) + $(SECTOR_COUNT_BT_STAGE2) )
-SECTOR_COUNT_KERNEL = $(shell cut -d' ' -f4 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_KERNEL = $(shell cut -d' ' -f4 configure 2> /dev/null || echo 30 )
 SECTOR_START_APP_TTT = $(shell expr $(SECTOR_START_KERNEL) + $(SECTOR_COUNT_KERNEL) )
-SECTOR_COUNT_APP_TTT = $(shell cut -d' ' -f5 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_APP_TTT = $(shell cut -d' ' -f5 configure 2> /dev/null || echo 30 )
 SECTOR_START_APP_CALC = $(shell expr $(SECTOR_START_APP_TTT) + $(SECTOR_COUNT_APP_TTT) )
-SECTOR_COUNT_APP_CALC = $(shell cut -d' ' -f6 configure 2> /dev/null || echo 0 )
+SECTOR_COUNT_APP_CALC = $(shell cut -d' ' -f6 configure 2> /dev/null || echo 30 )
+SECTOR_START_APP_LS =  $(shell expr $(SECTOR_START_APP_CALC) + $(SECTOR_COUNT_APP_CALC) )
+SECTOR_COUNT_APP_LS = $(shell cut -d' ' -f7 configure 2> /dev/null || echo 30 )
+SECTOR_START_APP_CAT =  $(shell expr $(SECTOR_START_APP_LS) + $(SECTOR_COUNT_APP_LS) )
+SECTOR_COUNT_APP_CAT = $(shell cut -d' ' -f7 configure 2> /dev/null || echo 30 )
 
 # configure file stores the sector size of each sub images.
-configure: $(bt_stage1) $(rm_static) $(bt_stage2) $(kernel_core) $(app_tic_tac_toe) $(app_calc)
+configure: $(bt_stage1) $(rm_static) $(bt_stage2) $(kernel_core) $(app_tic_tac_toe) $(app_calc) $(app_ls) $(app_cat)
 	bash scripts/build_image.sh /dev/null $^ > $@
 	rm -r $(BUILD_DIR)/ && "Cleared build directory" || echo "Build directory is clean."
 
-$(image_vmdk): $(bt_stage1) $(rm_static) $(bt_stage2) $(kernel_core) $(app_tic_tac_toe) $(app_calc)
+$(image_vmdk): $(bt_stage1) $(rm_static) $(bt_stage2) $(kernel_core) $(app_tic_tac_toe) $(app_calc) $(app_ls) $(app_cat) $(BUILD_DIR)/external/bin/mbr_builder $(BUILD_DIR)/external/example/sample_fs
 	test -s configure || { echo -e "\033[0;31mFailed! Please execute 'make configure' first.\033[0m" >&2; exit 1; }
-	bash scripts/build_image.sh $@ $^
+	bash scripts/build_image.sh $(BUILD_DIR)/temp_vmdk $(bt_stage1) $(rm_static) $(bt_stage2) $(kernel_core) $(app_tic_tac_toe) $(app_calc) $(app_ls) $(app_cat)
+	./$(BUILD_DIR)/external/bin/mbr_builder $@  $(BUILD_DIR)/temp_vmdk $(BUILD_DIR)/external/example/sample_fs
 	@echo "Image Size : $$(stat -c %s $@) byte(s)"
 
 clean:
 	rm -r $(BUILD_DIR)/ || echo "Build directory is clean."
 	rm -f configure
 
-
 include emulator/Makefile.mk
 
 include $(SRC_DIR)/usr/include/Makefile.mk
@@ -107,9 +124,12 @@ include $(SRC_DRIVERS)/disk/Makefile.mk
 include $(SRC_DRIVERS)/display/Makefile.mk
 include $(SRC_DRIVERS)/keyboard/Makefile.mk
 
+include $(SRC_DIR)/fs/Makefile.mk
+
 include $(SRC_LIB)/app/Makefile.mk
 include $(SRC_LIB_DS)/Makefile.mk
-include $(SRC_LIB_SYSCALL)/Makefile.mk
 include $(SRC_LIB_UTILS)/Makefile.mk
 
-include $(SRC_APP)/Makefile.mk
+include $(SRC_APP)/Makefile.mk
+
+include external/src/Makefile.mk

emulator/Makefile.mk

@@ -1,6 +1,7 @@
 QEMU_SHUT_FLAGS= -no-shutdown -no-reboot
 QEMU_EXTRA_FLAGS=
 QEMU_GDB_PORT=9000
+GDB_EX='echo Use GDB_EX="<command>" to execute command on connect.\n'
 
 qemu: $(image_vmdk)
 	qemu-system-x86_64 -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) $(QEMU_EXTRA_FLAGS)
@@ -12,4 +13,6 @@ qemu_debug: $(image_vmdk)
 	qemu-system-x86_64 -S -gdb tcp::$(QEMU_GDB_PORT) -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) -d  cpu,exec,in_asm
 
 qemu_debug_connect:
-	gdb -ex "target remote :$(QEMU_GDB_PORT)"
+	gdb -x $(ROOT_DIR)/.gdbinit -ex "target remote :$(QEMU_GDB_PORT)" -ex "$(GDB_EX)" -ex 'continue'
+
+

external/src/Makefile.mk

@@ -0,0 +1,18 @@
+external: $(BUILD_DIR)/external/bin/mkfs.ffs $(BUILD_DIR)/external/example/sample_fs
+
+$(BUILD_DIR)/external/bin/mkfs.ffs: external/src/mkfs_ffs.c
+	mkdir -p $(dir $@)
+	$(HOST_CC) -o $@ $<
+
+$(BUILD_DIR)/external/bin/mbr_builder: external/src/mbr_builder.c
+	mkdir -p $(dir $@)
+	$(HOST_CC) -o $@ $<
+
+$(BUILD_DIR)/external/out/minimal_fs: $(BUILD_DIR)/external/bin/mkfs.ffs $(wildcard tests/**/*)
+	mkdir -p $(dir $@)
+	$< tests/ $@
+
+$(BUILD_DIR)/external/example/sample_fs: $(BUILD_DIR)/external/out/minimal_fs $(BUILD_DIR)/external/bin/mkfs.ffs
+	mkdir -p $(dir $@)
+	echo "TODO: Append new files to filesystem"
+	cp -f $< $@

external/src/mbr_builder.c

@@ -0,0 +1,98 @@
+// Classical generic MBR
+// https://en.wikipedia.org/wiki/Master_boot_record#PTE
+#include <fuzzy/fs/mbr.h>
+#include<stdio.h>
+
+char DRIVE = 0x80;
+char DRIVE_INACTIVE = 0x00;
+// https://en.wikipedia.org/wiki/Partition_type
+char PARTITION_TYPE = 0x07;  // Stealing exFAT
+
+// https://en.wikipedia.org/wiki/Cylinder-head-sector
+void lba_to_chs(int lba, char chs[3]) {
+    // only lower 3 bytes of CHS are valid.
+    // TODO: Not implemented
+
+    // # Sector
+    // START_CHS_HEAD=
+    // START_CHS_CYLINDER=
+    // START_CHS_SECTOR
+    // START_CHS_ADDRESS=
+    // END_CHS_HEAD=
+    // END_CHS_CYLINDER=
+    // END_CHS_SECTORA
+    // END_CHS_ADDRESS=
+}
+
+void write_boot_signature(FILE *out) {
+    fseek(out, 510, SEEK_SET);
+    char boot_signature[2]={0x55, 0xAA};
+    fwrite(boot_signature, 1, sizeof(boot_signature), out);
+}
+
+void write_partition_entry(FILE *out, int id, char drive, int lba, int sector_count) {
+    struct PartitionEntry entry;
+    entry.drive = drive;
+    entry.partition_type = PARTITION_TYPE;
+    entry.lba = lba;
+    entry.sector_count = sector_count;
+    // TODO: write lba using lba_to_chs too
+    fseek(out, MBR_PARTITION_BEGIN+id*sizeof(struct PartitionEntry), SEEK_SET);
+    fwrite(&entry, 1, sizeof(entry), out);
+    printf("Added partition entry %d at lba: %d sector_count: %d\n",
+        id, lba, sector_count);
+}
+
+void write_partition(FILE *out, FILE *in, int lba) {
+    int count;
+    char buffer[1024];
+    rewind(in);
+    fseek(out, lba*512, SEEK_SET);
+    while ((count = fread(buffer, sizeof(char), sizeof(buffer), in)) > 0) {
+        fwrite(buffer, sizeof(char), count, out);
+    }
+}
+
+void populate_outfile_using_image_prefix(FILE *out, FILE *in) {
+    char buffer[1024];
+    int count;
+    rewind(in);
+    rewind(out);
+    while ((count = fread(buffer, sizeof(char), sizeof(buffer), in)) > 0) {
+        fwrite(buffer, sizeof(char), count, out);
+    }
+}
+
+int main(int argc, char *argv[]) {
+    // Meant to create single parition entry within MBR.
+    if (argc != 4) {
+        printf("Usage: %s <out_file> <image_prefix> <parition1>\n", argv[0]);
+        return 0;
+    }
+    FILE *out = fopen(argv[1], "wb");
+    FILE *image_prefix = fopen(argv[2], "rb");
+    FILE *part1 = fopen(argv[3], "rb");
+
+    populate_outfile_using_image_prefix(out, image_prefix);
+
+    int lba, sector_count;   // 4 bytes
+    {
+        fseek(out, 0L, SEEK_END);
+        int file_size = ftell(out);
+        rewind(out);
+        lba=(file_size+511)/512;
+    }
+    {
+        fseek(part1, 0L, SEEK_END);
+        int file_size = ftell(part1);
+        sector_count=(file_size+511)/512;
+    }
+
+    write_boot_signature(out);
+    write_partition_entry(out, 0, DRIVE, lba, sector_count);
+    write_partition(out, part1, lba);
+    write_partition_entry(out, 1, DRIVE_INACTIVE, 0, 0);
+    write_partition_entry(out, 2, DRIVE_INACTIVE, 0, 0);
+    write_partition_entry(out, 3, DRIVE_INACTIVE, 0, 0);
+    return 0;
+}