[microTVM] Update Zephyr 2.5 (apache#7786)

* update to zephyr 2.5 * unbreak test_zephyr * fix stack size * always create packer.log * add qemu debugging * fix transport with debug false * size down ring buf, shouldn't need to be so large * update to zephyr 2.5 * fix buffer size * cleanup * cleanup * remove debugger * nit * update ci script * remove debug mode * fix packer log * comment * update ci_qemu * change zephyr version on Vagrant * make it compatible to zephyr 2.4 Co-authored-by: Andrew Reusch <areusch@octoml.ai>
trevor-m · May 6, 2021 · d7e243d · d7e243d
1 parent 8f45c44
commit d7e243d
Show file tree

Hide file tree

Showing 14 changed files with 48 additions and 32 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -49,7 +49,7 @@ ci_gpu = "tlcpack/ci-gpu:v0.72"
 ci_cpu = "tlcpack/ci-cpu:v0.73"
 ci_wasm = "tlcpack/ci-wasm:v0.70"
 ci_i386 = "tlcpack/ci-i386:v0.72-t0"
-ci_qemu = "tlcpack/ci-qemu:v0.02"
+ci_qemu = "tlcpack/ci-qemu:v0.03"
 ci_arm = "tlcpack/ci-arm:v0.03"
 // <--- End of regex-scanned config.
 

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
@@ -205,12 +205,11 @@ def build_command(args):
         os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"),
         args.provider or ALL_PROVIDERS,
     )
-    env = None
+    env = copy.copy(os.environ)
     packer_args = ["packer", "build"]
+    env["PACKER_LOG"] = "1"
+    env["PACKER_LOG_PATH"] = "packer.log"
     if args.debug_packer:
-        env = copy.copy(os.environ)
-        env["PACKER_LOG"] = "1"
-        env["PACKER_LOG_PATH"] = "packer.log"
         packer_args += ["-debug"]
 
     packer_args += ["packer.json"]

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -16,7 +16,7 @@
 # under the License.
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "tlcpack/microtvm-zephyr"
+  config.vm.box = "tlcpack/microtvm-zephyr-2.5"
 
   tvm_home = "../../../.."
   dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())]

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
@@ -81,17 +81,20 @@ pip3 install --user -U west
 echo 'export PATH=$HOME/.local/bin:"$PATH"' >> ~/.profile
 source ~/.profile
 echo PATH=$PATH
-west init --mr v2.4.0 ~/zephyr
+west init --mr v2.5.0 ~/zephyr
 cd ~/zephyr
 west update
 west zephyr-export
 
 cd ~
 echo "Downloading zephyr SDK..."
-wget --no-verbose https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.11.3/zephyr-sdk-0.11.3-setup.run
-chmod +x zephyr-sdk-0.11.3-setup.run
-./zephyr-sdk-0.11.3-setup.run -- -d ~/zephyr-sdk -y
-rm -rf zephyr-sdk-0.11.3-setup.run
+ZEPHYR_SDK_VERSION=0.12.3
+ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run
+wget --no-verbose -O $ZEPHYR_SDK_FILE \
+    https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-x86_64-linux-setup.run
+chmod +x $ZEPHYR_SDK_FILE
+"./$ZEPHYR_SDK_FILE" -- -d ~/zephyr-sdk -y
+rm -rf ZEPHYR_SDK_FILE
 
 # GDB for Zephyr SDK depends on python3.8
 sudo add-apt-repository ppa:deadsnakes/ppa

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -2,11 +2,11 @@
     "stm32f746xx": {
         "vid_hex": "0483",
         "pid_hex": "374b",
-        "test_cmd": ["pytest", "tests/micro/qemu/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
+        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
         },
     "nrf5340dk": {
         "vid_hex": "1366",
         "pid_hex": "1055",
-        "test_cmd": ["pytest", "tests/micro/qemu/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
+        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
         }
 }
diff --git a/apps/microtvm/zephyr/demo_runtime/boards/nucleo_f746zg.conf b/apps/microtvm/zephyr/demo_runtime/boards/nucleo_f746zg.conf
@@ -20,8 +20,8 @@
 # For intrinsics used by generated optimized operators.
 CONFIG_CMSIS_DSP=y
 
-# Required for Cortex-M33 devices.
-CONFIG_MAIN_STACK_SIZE=50
+# For operations that stack allocates a large float array.
+CONFIG_MAIN_STACK_SIZE=1536
 
 # For random number generation.
 CONFIG_ENTROPY_GENERATOR=y

diff --git a/apps/microtvm/zephyr/demo_runtime/boards/qemu_x86.conf b/apps/microtvm/zephyr/demo_runtime/boards/qemu_x86.conf
@@ -21,3 +21,5 @@
 CONFIG_TEST_RANDOM_GENERATOR=y
 CONFIG_TIMER_RANDOM_GENERATOR=y
 
+# Default stack size is 1k, this is required for debug mode. 
+CONFIG_MAIN_STACK_SIZE=1536
diff --git a/apps/microtvm/zephyr/demo_runtime/prj.conf b/apps/microtvm/zephyr/demo_runtime/prj.conf
@@ -33,4 +33,3 @@ CONFIG_FPU=y
 
 # For TVMPlatformAbort().
 CONFIG_REBOOT=y
-
diff --git a/apps/microtvm/zephyr/demo_runtime/src/main.c b/apps/microtvm/zephyr/demo_runtime/src/main.c
@@ -127,18 +127,18 @@ tvm_crt_error_t TVMPlatformGenerateRandom(uint8_t* buffer, size_t num_bytes) {
   return kTvmErrorNoError;
 }
 
-// Memory pool for use by TVMPlatformMemoryAllocate.
-K_MEM_POOL_DEFINE(tvm_memory_pool, 64, 1024, 216, 4);
+// Heap for use by TVMPlatformMemoryAllocate.
+K_HEAP_DEFINE(tvm_heap, 216 * 1024);
 
 // Called by TVM to allocate memory.
 tvm_crt_error_t TVMPlatformMemoryAllocate(size_t num_bytes, DLDevice dev, void** out_ptr) {
-  *out_ptr = k_mem_pool_malloc(&tvm_memory_pool, num_bytes);
+  *out_ptr = k_heap_alloc(&tvm_heap, num_bytes, K_NO_WAIT);
   return (*out_ptr == NULL) ? kTvmErrorPlatformNoMemory : kTvmErrorNoError;
 }
 
 // Called by TVM to deallocate memory.
 tvm_crt_error_t TVMPlatformMemoryFree(void* ptr, DLDevice dev) {
-  k_free(ptr);
+  k_heap_free(&tvm_heap, ptr);
   return kTvmErrorNoError;
 }
 
@@ -214,7 +214,11 @@ tvm_crt_error_t TVMPlatformTimerStop(double* elapsed_time_seconds) {
 }
 
 // Ring buffer used to store data read from the UART on rx interrupt.
+#if BOARD == qemu_x86
 #define RING_BUF_SIZE_BYTES 4 * 1024
+#else
+#define RING_BUF_SIZE_BYTES 1 * 1024
+#endif
 RING_BUF_DECLARE(uart_rx_rbuf, RING_BUF_SIZE_BYTES);
 
 // Small buffer used to read data from the UART into the ring buffer.

diff --git a/docker/install/ubuntu_install_zephyr.sh b/docker/install/ubuntu_install_zephyr.sh
@@ -55,7 +55,7 @@ pip3 install west
 #EOF
 #chmod a+x /usr/local/bin/west
 
-west init --mr v2.4.0 /opt/zephyrproject
+west init --mr v2.5.0 /opt/zephyrproject
 cd /opt/zephyrproject
 west update
 
@@ -72,8 +72,9 @@ west zephyr-export
 #/opt/west/bin/pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt
 pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt
 
-SDK_VERSION=0.11.3
-wget --no-verbose \
-     https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${SDK_VERSION}/zephyr-sdk-${SDK_VERSION}-setup.run
-chmod +x zephyr-sdk-${SDK_VERSION}-setup.run
-./zephyr-sdk-${SDK_VERSION}-setup.run -- -d /opt/zephyr-sdk
+ZEPHYR_SDK_VERSION=0.12.3
+ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run
+wget --no-verbose -O $ZEPHYR_SDK_FILE \
+    https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-x86_64-linux-setup.run
+chmod +x $ZEPHYR_SDK_FILE
+"./$ZEPHYR_SDK_FILE" -- -d /opt/zephyr-sdk
diff --git a/python/tvm/micro/contrib/zephyr.py b/python/tvm/micro/contrib/zephyr.py
@@ -653,7 +653,7 @@ def popen_kwargs(self):
         args = dict(
             args=self._west_cmd
             + [
-                "attach",
+                "debug",
                 "--skip-rebuild",
                 "--build-dir",
                 self._build_dir,

diff --git a/python/tvm/micro/transport/file_descriptor.py b/python/tvm/micro/transport/file_descriptor.py
@@ -86,8 +86,13 @@ def read(self, n, timeout_sec):
 
         end_time = None if timeout_sec is None else time.monotonic() + timeout_sec
 
-        self._await_ready([self.read_fd], [], end_time=end_time)
-        to_return = os.read(self.read_fd, n)
+        while True:
+            self._await_ready([self.read_fd], [], end_time=end_time)
+            try:
+                to_return = os.read(self.read_fd, n)
+                break
+            except BlockingIOError:
+                pass
 
         if not to_return:
             self.close()

diff --git a/python/tvm/micro/transport/serial.py b/python/tvm/micro/transport/serial.py
@@ -92,7 +92,10 @@ def close(self):
     def read(self, n, timeout_sec):
         if timeout_sec is None:
             self._port.timeout = None
-            return self._port.read(n)
+            in_waiting = self._port.in_waiting
+            if in_waiting > 0:
+                return self._port.read(min(n, in_waiting))
+            return self._port.read(1)
 
         end_time = time.monotonic() + timeout_sec
         to_return = bytearray()

diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py
@@ -59,10 +59,10 @@ def _make_sess_from_op(model, zephyr_board, west_cmd, op_name, sched, arg_bufs):
 
 
 def _make_session(model, target, zephyr_board, west_cmd, mod):
-    test_name = f"{os.path.splitext(os.path.abspath(__file__))[0]}-{model}"
+    test_name = f"{os.path.splitext(os.path.abspath(__file__))[0]}_{model}"
     prev_build = f"{test_name}-last-build.micro-binary"
     workspace_root = (
-        f'{test_name}-workspace/{datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")}'
+        f'{test_name}_workspace/{datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")}'
     )
     workspace_parent = os.path.dirname(workspace_root)
     if not os.path.exists(workspace_parent):