Add RV32 support (keystone-enclave#213)

* Add RV32 Linux/Buildroot
* Modify bootrom
* Add RV32-related scripts (source32.sh, rv32-setup.sh)
* Bump SDK to be able to compile with RV32
* Bump driver
* Bump riscv-pk
* Bump QEMU
* Create document (Running-Keystone-on-RV32.rst)

Co-authored-by: Dayeol Lee <dayeol@berkeley.edu>

.prebuilt_tools_shasums

@@ -1,2 +1,3 @@
 05211edea5a47ebaf906ef4bed2c9609a93e8c6ad5e45f8c87678eabbfd424e7  1.0.tar.gz
 bdc9e3ec47ac461ecc7865609fda6b820439c36130e9da9275af010d9f4fe4bc  2.0.tar.gz
+e73fa1191ce97be2531401d4628384c321c45ca5e4c83756e8415f2cc31a0b18  rv32gc.tar.gz

CMakeLists.txt

@@ -155,9 +155,11 @@ add_custom_command(DEPENDS ${buildroot_config} OUTPUT ${buildroot_wrkdir}/.confi
 ###############################################################################
 
 add_custom_command(OUTPUT ${bootrom_wrkdir} COMMAND mkdir -p ${bootrom_wrkdir})
-add_custom_target("bootrom" ALL
-  COMMAND $(MAKE) -C ${bootrom_srcdir} O=${bootrom_wrkdir}
-  DEPENDS ${bootrom_wrkdir} ${bootrom_srcdir}
+add_custom_target("bootrom-sync" DEPENDS ${bootrom_srcdir} ${bootrom_wrkdir}
+  COMMAND rsync -r ${bootrom_srcdir}/ ${bootrom_wrkdir})
+add_custom_target("bootrom" ALL DEPENDS ${bootrom_wrkdir} ${bootrom_srcdir} "bootrom-sync"
+  COMMAND $(MAKE) -C ${bootrom_wrkdir} O=${bootrom_wrkdir} march=${ISA} mabi=${ABI} CC=riscv${BITS}-unknown-elf-gcc
+    OBJCOPY=riscv${BITS}-unknown-elf-objcopy
   COMMENT "Building bootrom"
 )
 

bootrom/Makefile

@@ -1,7 +1,10 @@
-CC = riscv64-unknown-elf-gcc
-OBJCOPY= riscv64-unknown-elf-objcopy
+march ?= rv64imafdc
+mabi ?= lp64
+CC ?= riscv64-unknown-elf-gcc
+OBJCOPY ?= riscv64-unknown-elf-objcopy
+
 CFLAGS = \
-	-march=rv64imafdc -mcmodel=medany -mabi=lp64 \
+	-march=$(march) -mcmodel=medany -mabi=$(mabi) \
 	-nostdlib -nostartfiles -fno-common -std=gnu11 \
 	-static \
 	-fPIC \

bootrom/bootloader.S

@@ -4,35 +4,43 @@
 #define DTB   0
 #define INIT  8
 
+#if __riscv_xlen == 64
+# define STORE    sd
+# define LOAD     ld
+#else
+# define STORE    sw
+# define LOAD     lw
+#endif
+
 reset:
   li sp, STACK_TOP
   addi sp, sp, -16
 
-  sd x0, INIT(sp)
+  STORE x0, INIT(sp)
 
   csrr t0, mhartid
   bnez t0, loop
-  
+
   # sd a1, DTB(sp)
-  
+
   /* secure boot */
   call bootloader
 
   /* clear_stack_from_bottom */
   li t1, STACK_BOTTOM
 clean_loop:
   beq t1, sp, clean_done
-  sd x0, (t1)
+  STORE x0, (t1)
   addi t1, t1, 8
   j clean_loop
 clean_done:
   li t0, 1
-  sd t0, INIT(sp)
+  STORE t0, INIT(sp)
   fence
 
 boot:
   call clear_all_but_sp
-  
+
   # boot payload
 	csrr a0, mhartid
   la a1, _dtb
@@ -41,7 +49,7 @@ boot:
   jr t0
 
 loop:
-  ld t1, INIT(sp)
+  LOAD t1, INIT(sp)
   beqz t1, loop
   j boot
 

conf/linux32-defconfig

@@ -19,7 +19,7 @@ CONFIG_MODULES=y
 CONFIG_CMA=y
 CONFIG_CMA_AREAS=7
 CONFIG_DMA_CMA=y
-CONFIG_CMA_SIZE_MBYTES=1024
+CONFIG_CMA_SIZE_MBYTES=512
 CONFIG_CMA_SIZE_SEL_MBYTES=y
 CONFIG_CMA_ALIGNMENT-9
 CONFIG_MODULE_UNLOAD=y
@@ -39,6 +39,7 @@ CONFIG_PCIEPORTBUS=y
 CONFIG_PCI_HOST_GENERIC=y
 CONFIG_PCIE_XILINX=y
 CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_VIRTIO_BLK=y
 CONFIG_BLK_DEV_SD=y

conf/qemu_riscv32_virt_defconfig

@@ -32,7 +32,7 @@ BR2_TARGET_ROOTFS_EXT2=y
 BR2_ROOTFS_OVERLAY=""
 
 # Linux headers same as kernel, a 5.1 series
-BR2_PACKAGE_HOST_LINUX_HEADERS_CUSTOM_5_4=y
+BR2_PACKAGE_HOST_LINUX_HEADERS_CUSTOM_5_1=y
 
 # Kernel
 # BR2_LINUX_KERNEL=y

docs/source/Getting-Started/Running-Keystone-on-RV32.rst

@@ -0,0 +1,54 @@
+.. 
+
+Running Keystone in RV32
+--------------------------------------
+
+Currently we support running Keystone using all three priviledges, which are Machine, Supervisor, and User mode (M/S/U). Currently, we are working on a Machine and User mode \
+implementation on Keystone to better support embedded systems. 
+
+Setup RV32 Environment
+#############################
+
+First, run the ``rv32-setup.sh`` script (located at the top-level directory) to install the RV32 toolchain (which uses the GC extensions). 
+
+::
+
+  ./rv32-setup.sh
+  
+This script will unzip and install the RV32 toolchain and store it in ``./riscv32``. The script will also set the ``$RISCV`` environment variable to point to the ``riscv32`` \
+directory. We strongly recommend adding the following to your ``.bashrc``.
+
+::
+
+  export RISCV=$(pwd)/riscv32
+  export PATH=$RISCV/bin:$PATH
+
+Build SDK in RV32
+#############################
+
+We will now have to build the SDK in RV32. Go to your SDK directory and create a build directory. Remember to set your SDK path
+
+::
+
+  mkdir <build directory>
+  cd <build directory>
+  export KEYSTONE_SDK_DIR=<install_directory>
+  cmake .. -DRISCV32=y 
+  make
+  make install
+  
+This will build the SDK in RV32. 
+
+Build Keystone in RV32
+#############################
+
+Similar to the SDK, we may now build the entire Keystone framework in RV32. First, create a build directory and specify the RV32 flag.
+
+::
+
+  mkdir <build directory>
+  cd <build directory>
+  cmake .. -DRISCV32=y 
+  make
+
+This will begin building Keystone in RV32. 

patches/linux/linux32.patch

@@ -1,38 +1,39 @@
 diff --git arch/riscv/Kconfig arch/riscv/Kconfig
-index eb56c82d8a..43dd2680c6 100644
+index 59a4727ecd..b9154739b3 100644
 --- arch/riscv/Kconfig
 +++ arch/riscv/Kconfig
-@@ -11,6 +11,7 @@
-
+@@ -12,6 +12,7 @@ config 32BIT
+ 
  config RISCV
-     def_bool y
-+    select ARCH_32BIT_OFF_T if !64BIT
-     # even on 32-bit, physical (and DMA) addresses are > 32-bits
-     select PHYS_ADDR_T_64BIT
-     select OF
+ 	def_bool y
++  select ARCH_32BIT_OFF_T if !64BIT
+ 	# even on 32-bit, physical (and DMA) addresses are > 32-bits
+ 	select PHYS_ADDR_T_64BIT
+ 	select OF
 diff --git arch/riscv/include/uapi/asm/unistd.h arch/riscv/include/uapi/asm/unistd.h
-index 0e2eeeb1fd..486a288b45 100644
+index 13ce76cc5a..95217ebb63 100644
 --- arch/riscv/include/uapi/asm/unistd.h
 +++ arch/riscv/include/uapi/asm/unistd.h
-@@ -17,8 +17,11 @@
-
+@@ -17,9 +17,12 @@
+ 
  #ifdef __LP64__
  #define __ARCH_WANT_NEW_STAT
 -#define __ARCH_WANT_SET_GET_RLIMIT
+ #define __ARCH_WANT_SYS_CLONE3
  #endif /* __LP64__ */
 +#define __ARCH_WANT_SET_GET_RLIMIT
 +#ifndef __LP64__
 +#define __ARCH_WANT_TIME32_SYSCALLS
 +#endif
-
+ 
  #include <asm-generic/unistd.h>
-
+ 
 diff --git arch/riscv/kernel/vdso/Makefile arch/riscv/kernel/vdso/Makefile
-index fec62b24df..eed1c137f6 100644
+index 49a5852fd0..725dfe8c93 100644
 --- arch/riscv/kernel/vdso/Makefile
 +++ arch/riscv/kernel/vdso/Makefile
-@@ -2,11 +2,9 @@
-
+@@ -3,11 +3,9 @@
+ 
  # Symbols present in the vdso
  vdso-syms  = rt_sigreturn
 -ifdef CONFIG_64BIT
@@ -42,25 +43,22 @@ index fec62b24df..eed1c137f6 100644
 -endif
  vdso-syms += getcpu
  vdso-syms += flush_icache
-
+ 
 diff --git arch/riscv/mm/init.c arch/riscv/mm/init.c
-index 573463d..be1f2c4 100644
+index 42bf939693..49d7da7d86 100644
 --- arch/riscv/mm/init.c
 +++ arch/riscv/mm/init.c
-@@ -19,6 +19,8 @@
- #include <asm/pgtable.h>
- #include <asm/io.h>
-
+@@ -11,6 +11,7 @@
+ #include <linux/swap.h>
+ #include <linux/sizes.h>
+ #include <linux/of_fdt.h>
 +#include <linux/dma-contiguous.h>
-+
- #include "../kernel/head.h"
 
- unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
-@@ -457,6 +459,7 @@ void __init paging_init(void)
- 	sparse_init();
+ #include <asm/fixmap.h>
+ #include <asm/tlbflush.h>
+@@ -444,4 +445,5 @@ void __init paging_init(void)
+ 	setup_vm_final();
  	setup_zero_page();
  	zone_sizes_init();
 +	dma_contiguous_reserve(memblock_end_of_DRAM());
  }
-
- #ifdef CONFIG_SPARSEMEM_VMEMMAP

rv32-setup.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+echo "Starting RV32 Instllation..."
+
+if ( $(command -v riscv32-unknown-linux-gnu-gcc > /dev/null) &&
+  $(command -v riscv32-unknown-elf-gcc > /dev/null) )
+then
+  echo "RISCV32 tools are already installed"
+else
+  echo "Downloading Prebuilt RISC-V 32 Toolchain... "
+
+  export RISCV32=$(pwd)/riscv32
+  export PATH=$PATH:$RISCV32/bin
+  wget https://keystone-enclave.eecs.berkeley.edu/files/rv32gc.tar.gz
+
+  # Check tool integrity
+  echo "Verifying prebuilt toolchain integrity..."
+  sha256sum -c .prebuilt_tools_shasums --status --ignore-missing
+  if [[ $? != 0 ]]
+  then
+      echo "Toolchain binary download incomplete or corrupted. You can build the toolchain locally or try again."
+      exit 1
+  fi
+
+  tar -xzvf rv32gc.tar.gz
+  mv riscv32gc riscv32
+  rm rv32gc.tar.gz
+  echo "Toolchain has been installed in $RISCV32"
+fi

source32.sh

@@ -0,0 +1,3 @@
+export RISCV=$(pwd)/riscv32
+export PATH=$RISCV/bin:$PATH
+export KEYSTONE_SDK_DIR=$(pwd)/sdk/build