[microTVM][autoTVM] Follow up fixes to apache#9003 (apache#9018)

* fix test and cleanup

* fix tutorial doc

* fix verbose for tutorial

* fix tune check

* address comments

* address comments

python/tvm/micro/session.py

@@ -87,6 +87,8 @@ def __init__(
         self._rpc = None
         self._graph_executor = None
 
+        self._exit_called = False
+
     def get_system_lib(self):
         return self._rpc.get_function("runtime.SystemLib")()
 
@@ -130,7 +132,7 @@ def __enter__(self):
                     int(timeouts.session_start_retry_timeout_sec * 1e6),
                     int(timeouts.session_start_timeout_sec * 1e6),
                     int(timeouts.session_established_timeout_sec * 1e6),
-                    self._shutdown,
+                    self._cleanup,
                 )
             )
             self.device = self._rpc.cpu(0)
@@ -142,9 +144,11 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
         """Tear down this session and associated RPC session resources."""
-        self.transport.__exit__(exc_type, exc_value, exc_traceback)
+        if not self._exit_called:
+            self._exit_called = True
+            self.transport.__exit__(exc_type, exc_value, exc_traceback)
 
-    def _shutdown(self):
+    def _cleanup(self):
         self.__exit__(None, None, None)
 
 
@@ -289,6 +293,6 @@ def compile_and_create_micro_session(
     transport = generated_project.transport()
 
     rpc_session = Session(transport_context_manager=transport)
-    # RPC exit is called by shutdown function.
+    # RPC exit is called by cleanup function.
     rpc_session.__enter__()
     return rpc_session._rpc._sess

python/tvm/micro/testing.py

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Defines the test methods used with microTVM."""
+
+import pathlib
+import json
+from typing import Union
+
+
+def check_tune_log(log_path: Union[pathlib.Path, str]):
+    """Read the tuning log and check each result."""
+    with open(log_path, "r") as f:
+        lines = f.readlines()
+
+    for line in lines:
+        if len(line) > 0:
+            tune_result = json.loads(line)
+            assert tune_result["result"][0][0] < 1000000000.0

src/runtime/crt/host/Makefile

@@ -28,7 +28,11 @@ CC ?= ${PREFIX}gcc
 CXX ?= ${PREFIX}g++
 RANLIB ?= ${PREFIX}ranlib
 
+ifeq (${VERBOSE}, 1)
+QUIET ?= 
+else
 QUIET ?= @
+endif
 
 PWD = $(shell pwd)
 BUILD_DIR = build
@@ -38,7 +42,7 @@ CRT_LIBS = $(patsubst %, $(BUILD_DIR)/crt/lib%.a, $(CRT_LIB_NAMES))
 CRT_INCLUDES = $(glob crt/include/**)
 
 $(BUILD_DIR)/crt/lib%.a: $(glob crt/src/runtime/%/*.c)
-	${QUIET}cd crt && $(MAKE) \
+	${QUIET}cd crt && $(MAKE) -s \
 		BUILD_DIR=../$(BUILD_DIR)/crt \
 		CRT_CONFIG=$(PWD)/crt_config/crt_config.h \
 		EXTRA_CFLAGS="$(CFLAGS)" \

src/runtime/crt/host/microtvm_api_server.py

@@ -51,7 +51,11 @@ def server_info_query(self, tvm_version):
             model_library_format_path=""
             if IS_TEMPLATE
             else PROJECT_DIR / MODEL_LIBRARY_FORMAT_RELPATH,
-            project_options=[server.ProjectOption("verbose", help="Run make with verbose output")],
+            project_options=[
+                server.ProjectOption(
+                    "verbose", help="Run make with verbose output", choices=(True, False)
+                )
+            ],
         )
 
     # These files and directories will be recursively copied into generated projects from the CRT.
@@ -111,7 +115,7 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec
     def build(self, options):
         args = ["make"]
         if options.get("verbose"):
-            args.append("QUIET=")
+            args.append("VERBOSE=1")
 
         args.append(self.BUILD_TARGET)
 

src/runtime/rpc/rpc_endpoint.cc

@@ -684,20 +684,21 @@ void RPCEndpoint::Init() {
 
 /*!
  * \brief Create a new RPCEndpoint instance.
- * \param channel RPCChannel used to communicate
+ * \param channel RPCChannel used to communicate.
  * \param name Name of this session, used to identify log messages from this RPCEndpoint instance.
- * \param The remote key reported during protocol initialization, or "%toinit" if the RPCEndpoint
- *     should handle this phase of the protocol for you. Some servers may prefer to access parts of
- *     the key to modify their behavior.
+ * \param remote_key The remote key reported during protocol initialization, or "%toinit" if the
+ * RPCEndpoint should handle this phase of the protocol for you. Some servers may prefer to access
+ * parts of the key to modify their behavior.
+ * \param fcleanup The cleanup Packed function.
  */
 std::shared_ptr<RPCEndpoint> RPCEndpoint::Create(std::unique_ptr<RPCChannel> channel,
                                                  std::string name, std::string remote_key,
-                                                 TypedPackedFunc<void()> fshutdown) {
+                                                 TypedPackedFunc<void()> fcleanup) {
   std::shared_ptr<RPCEndpoint> endpt = std::make_shared<RPCEndpoint>();
   endpt->channel_ = std::move(channel);
   endpt->name_ = std::move(name);
   endpt->remote_key_ = std::move(remote_key);
-  endpt->fshutdown_ = fshutdown;
+  endpt->fcleanup_ = fcleanup;
   endpt->Init();
   return endpt;
 }
@@ -736,7 +737,7 @@ void RPCEndpoint::ServerLoop() {
     (*f)();
   }
   channel_.reset(nullptr);
-  if (fshutdown_ != nullptr) fshutdown_();
+  if (fcleanup_ != nullptr) fcleanup_();
 }
 
 int RPCEndpoint::ServerAsyncIOEventHandler(const std::string& in_bytes, int event_flag) {

src/runtime/rpc/rpc_endpoint.h

@@ -161,13 +161,13 @@ class RPCEndpoint {
    * \param channel The communication channel.
    * \param name The local name of the session, used for debug
    * \param remote_key The remote key of the session
-   * \param fshutdown The shutdown Packed function
    *   if remote_key equals "%toinit", we need to re-intialize
    *   it by event handler.
+   * \param fcleanup The cleanup Packed function.
    */
   static std::shared_ptr<RPCEndpoint> Create(std::unique_ptr<RPCChannel> channel, std::string name,
                                              std::string remote_key,
-                                             TypedPackedFunc<void()> fshutdown = nullptr);
+                                             TypedPackedFunc<void()> fcleanup = nullptr);
 
  private:
   class EventHandler;
@@ -192,8 +192,8 @@ class RPCEndpoint {
   std::string name_;
   // The remote key
   std::string remote_key_;
-  // The shutdown Packed Function
-  TypedPackedFunc<void()> fshutdown_;
+  // Invoked when the RPC session is terminated
+  TypedPackedFunc<void()> fcleanup_;
 };
 
 /*!

tests/micro/zephyr/test_zephyr.py

@@ -21,6 +21,7 @@
 import subprocess
 import sys
 import logging
+import json
 
 import pytest
 import numpy as np
@@ -38,6 +39,8 @@
 from tvm.relay.expr_functor import ExprMutator
 from tvm.relay.op.annotation import compiler_begin, compiler_end
 
+from tvm.micro.testing import check_tune_log
+
 import conftest
 
 _LOG = logging.getLogger(__name__)
@@ -430,13 +433,18 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug):
             "project_type": "host_driven",
         },
     )
+
+    timeout = 200
     builder = tvm.autotvm.LocalBuilder(
+        timeout=timeout,
         n_parallel=1,
         build_kwargs={"build_option": {"tir.disable_vectorize": True}},
         do_fork=True,
         build_func=tvm.micro.autotvm_build_func,
     )
-    runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader)
+    runner = tvm.autotvm.LocalRunner(
+        number=1, repeat=1, timeout=timeout, module_loader=module_loader
+    )
 
     measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner)
 
@@ -456,8 +464,9 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug):
             ],
             si_prefix="M",
         )
+        assert tuner.best_flops > 0
 
-    assert tuner.best_flops > 0
+    check_tune_log(log_path)
 
     # Build without tuning
     with pass_context:

tests/python/unittest/test_crt.py

@@ -15,18 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import contextlib
-import copy
-import glob
 import os
 import pathlib
 import pytest
 import shutil
+import json
 
 pytest.importorskip("pty")
 import sys
-import subprocess
-import textwrap
 
 import numpy as np
 import pytest
@@ -39,6 +35,8 @@
 from tvm.topi.utils import get_const_tuple
 from tvm.topi.testing import conv2d_nchw_python
 
+from tvm.micro.testing import check_tune_log
+
 BUILD = True
 DEBUG = False
 
@@ -291,8 +289,9 @@ def test_autotune():
             ],
             si_prefix="M",
         )
+        assert tuner.best_flops > 0
 
-    assert tuner.best_flops > 0
+    check_tune_log(tune_log_file)
 
     # Build without tuning
     with pass_context:

tutorials/micro/micro_autotune.py

@@ -20,7 +20,9 @@
 
 Autotuning with micro TVM
 =========================
-**Author**: `Andrew Reusch <https://github.com/areusch>`_, `Mehrdad Hessar <https://github.com/mehrdadh>`
+**Authors**:
+`Andrew Reusch <https://github.com/areusch>`_,
+`Mehrdad Hessar <https://github.com/mehrdadh>`_
 
 This tutorial explains how to autotune a model using the C runtime.
 """
@@ -117,7 +119,7 @@
 
 module_loader = tvm.micro.AutoTvmModuleLoader(
     template_project_dir=repo_root / "src" / "runtime" / "crt" / "host",
-    project_options={},
+    project_options={"verbose": False},
 )
 builder = tvm.autotvm.LocalBuilder(
     n_parallel=1,
@@ -136,7 +138,7 @@
 #        project_options={
 #            "zephyr_board": BOARD,
 #            "west_cmd": "west",
-#            "verbose": 1,
+#            "verbose": False,
 #            "project_type": "host_driven",
 #        },
 #    )
@@ -147,8 +149,8 @@
 #        build_func=tvm.micro.autotvm_build_func,
 #    )
 #    runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader)
-
-# measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner)
+#
+#    measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner)
 
 ################
 # Run Autotuning
@@ -181,7 +183,10 @@
 temp_dir = tvm.contrib.utils.tempdir()
 
 project = tvm.micro.generate_project(
-    str(repo_root / "src" / "runtime" / "crt" / "host"), lowered, temp_dir / "project"
+    str(repo_root / "src" / "runtime" / "crt" / "host"),
+    lowered,
+    temp_dir / "project",
+    {"verbose": False},
 )
 
 # Compiling for physical hardware
@@ -193,7 +198,7 @@
 #        {
 #            "zephyr_board": BOARD,
 #            "west_cmd": "west",
-#            "verbose": 1,
+#            "verbose": False,
 #            "project_type": "host_driven",
 #        },
 #    )
@@ -221,7 +226,10 @@
 temp_dir = tvm.contrib.utils.tempdir()
 
 project = tvm.micro.generate_project(
-    str(repo_root / "src" / "runtime" / "crt" / "host"), lowered_tuned, temp_dir / "project"
+    str(repo_root / "src" / "runtime" / "crt" / "host"),
+    lowered_tuned,
+    temp_dir / "project",
+    {"verbose": False},
 )
 
 # Compiling for physical hardware
@@ -233,7 +241,7 @@
 #        {
 #            "zephyr_board": BOARD,
 #            "west_cmd": "west",
-#            "verbose": 1,
+#            "verbose": False,
 #            "project_type": "host_driven",
 #        },
 #    )