implemented micro_common and added Python interfaces

Author: mutinifni

python/tvm/__init__.py

@@ -42,7 +42,7 @@
 
 from . import ndarray as nd
 from .ndarray import context, cpu, gpu, opencl, cl, vulkan, metal, mtl
-from .ndarray import vpi, rocm, opengl, ext_dev
+from .ndarray import vpi, rocm, opengl, ext_dev, micro_dev
 
 from ._ffi.runtime_ctypes import TypeCode, TVMType
 from ._ffi.ndarray import TVMContext

python/tvm/contrib/binutil.py

@@ -0,0 +1,153 @@
+"""Utilities for binary file manipulation"""
+import subprocess
+from os.path import join, exists
+from . import util
+from .._ffi.base import py_str
+from ..api import register_func, convert
+
+
+@register_func("tvm_get_section_size")
+def tvm_get_section_size(binary_name, section):
+    """Finds size of the section in the binary.
+    Assumes "size" shell command exists (typically works only on Linux machines)
+
+    Parameters
+    ----------
+    binary_name : string
+        name of the binary file
+
+    section : string
+        type of section
+
+    Return
+    ------
+    size : integer 
+        size of the section in bytes
+    """
+    section_map = {"text": "1", "data": "2", "bss": "3"}
+    p1 = subprocess.Popen(["size", binary_name], stdout=subprocess.PIPE)
+    p2 = subprocess.Popen(["awk", "{print $" + section_map[section] + "}"],
+                           stdin=p1.stdout, stdout=subprocess.PIPE)
+    p3 = subprocess.Popen(["tail", "-1"], stdin=p2.stdout, stdout=subprocess.PIPE)
+    p1.stdout.close()
+    p2.stdout.close()
+    (out, _) = p3.communicate()
+    if p3.returncode != 0:
+        msg = "Error in finding section size:\n"
+        msg += py_str(out)
+        raise RuntimeError(msg)
+    return int(out)
+
+
+@register_func("tvm_relocate_binary")
+def tvm_relocate_binary(binary_name, text, data, bss):
+    """Relocates sections in the binary to new addresses
+
+    Parameters
+    ----------
+    binary_name : string
+        name of the binary file
+
+    text : string
+        text section address
+
+    data : string
+        data section address
+
+    bss : string
+        bss section address
+
+    Return
+    ------
+    rel_bin : bytearray
+        the relocated binary
+    """
+    tmp_dir = util.tempdir()
+    rel_obj = tmp_dir.relpath("relocated.o")
+    p1 = subprocess.Popen(["ld", binary_name,
+                           "-Ttext", text,
+                           "-Tdata", data,
+                           "-Tbss", bss,
+                           "-o", rel_obj],
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT)
+    (out, _) = p1.communicate()
+    if p1.returncode != 0:
+        msg = "Linking error using ld:\n"
+        msg += py_str(out)
+        raise RuntimeError(msg)
+    rel_bin = bytearray(open(rel_obj, "rb").read())
+    return rel_bin
+
+
+@register_func("tvm_read_binary_section")
+def tvm_read_binary_section(binary_name, section):
+    """Returns the contents of the specified section in the binary file
+
+    Parameters
+    ----------
+    binary_name : string 
+        name of the binary file
+
+    section : string
+        type of section
+
+    Return
+    ------
+    section_bin : bytearray
+        contents of the read section
+    """
+    tmp_dir = util.tempdir()
+    tmp_section = tmp_dir.relpath("tmp_section.bin")
+    p1 = subprocess.Popen(["objcopy", "--dump-section",
+                           "." + section + "=" + tmp_section,
+                           binary_name],
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT)
+    (out, _) = p1.communicate()
+    if p1.returncode != 0:
+        msg = "Error in using objcopy:\n"
+        msg += py_str(out)
+        raise RuntimeError(msg)
+    try:
+        # get section content if it exits
+        section_bin = bytearray(open(tmp_section, "rb").read())
+    except IOError:
+        # return empty bytearray if the section does not exist
+        section_bin = bytearray("")
+    return section_bin
+
+
+@register_func("tvm_get_symbol_map")
+def tvm_get_symbol_map(binary):
+    """Obtains a map of symbols to addresses in the passed binary
+
+    Parameters
+    ----------
+    binary : bytearray
+        the object file
+
+    Return
+    ------
+    symbol_map : dictionary
+        map of defined symbols to addresses
+    """
+    tmp_dir = util.tempdir()
+    tmp_obj = tmp_dir.relpath("tmp_obj.bin")
+    with open(tmp_obj, "wb") as out_file:
+        out_file.write(bytes(binary))
+    p1 = subprocess.Popen(["nm", "-C", "--defined-only", tmp_obj],
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT)
+    (out, _) = p1.communicate()
+    if p1.returncode != 0:
+        msg = "Error in using nm:\n"
+        msg += py_str(out)
+        raise RuntimeError(msg)
+    out = out.splitlines()
+    map_str = ""
+    for line in out:
+        line = line.split()
+        map_str += line[2] + "\n"
+        map_str += line[0] + "\n"
+    return map_str

python/tvm/micro/__init__.py

@@ -0,0 +1,8 @@
+"""uTVM module for bare-metal backends.
+
+uTVM (or the micro backend) enables provides support for bare-metal devices.
+Its targets currently include a host-emulated device which is used for testing,
+and JTAG-based openocd device which allows actual interfacing with microdevices.
+"""
+
+from ..contrib import binutil

python/tvm/ndarray.py

@@ -189,6 +189,22 @@ def ext_dev(dev_id=0):
     return TVMContext(12, dev_id)
 
 
+def micro_dev(dev_id=0):
+    """Construct a micro device
+
+    Parameters
+    ----------
+    dev_id : int, optional
+        The integer device id
+
+    Returns
+    -------
+    ctx : TVMContext
+        The created context
+    """
+    return TVMContext(13, dev_id)
+
+
 cl = opencl
 mtl = metal
 

src/runtime/micro/micro_common.cc

@@ -1,11 +1,15 @@
 /*!
  *  Copyright (c) 2019 by Contributors
- * \file bin_util.cc
- * \brief binary modification utilities
+ * \file micro_common.cc
+ * \brief common utilties for uTVM
  */
 
-#include <stdio.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/registry.h>
+#include <cstdio>
 #include <string>
+#include <sstream>
+#include <cstdint>
 #include "micro_session.h"
 #include "micro_common.h"
 
@@ -25,30 +29,75 @@ const char* SectionToString(SectionKind section) {
   }
 }
 
-// TODO: implement these in Python using PackedFunc + Registry
 void* GetSymbol(std::unordered_map<std::string, void*> symbol_map,
                 std::string name,
                 void* base_addr) {
-  return nullptr;
+  void* symbol_addr = symbol_map[name];
+  return (void*)((uint8_t*) symbol_addr - (uint8_t*) base_addr);
+}
+
+static std::string AddrToString(void* addr) {
+  std::stringstream stream;
+  if (addr != nullptr)
+    stream << addr;
+  else
+    stream << "0x0";
+  std::string string_addr = stream.str();
+  return string_addr;
 }
 
 std::string RelocateBinarySections(std::string binary_name,
                                    void* text,
                                    void* data,
                                    void* bss) {
-  return "";
+  const auto* f = Registry::Get("tvm_relocate_binary");
+  CHECK(f != nullptr) << "Require tvm_relocate_binary to exist in registry";
+  std::string relocated_bin = (*f)(binary_name,
+                                   AddrToString(text),
+                                   AddrToString(data),
+                                   AddrToString(bss));
+  return relocated_bin;
 }
 
 std::string ReadSection(std::string binary_name, SectionKind section) {
-  return "";
+  CHECK(section == kText || section == kData || section == kBss)
+    << "ReadSection requires section to be one of text, data or bss.";
+  const auto* f = Registry::Get("tvm_read_binary_section");
+  CHECK(f != nullptr) << "Require tvm_read_binary_section to exist in registry";
+  std::string section_contents = (*f)(binary_name, SectionToString(section));
+  return section_contents;
 }
 
 size_t GetSectionSize(std::string binary_name, SectionKind section) {
-  return 0;
+  CHECK(section == kText || section == kData || section == kBss)
+    << "GetSectionSize requires section to be one of text, data or bss.";
+  const auto* f = Registry::Get("tvm_get_section_size");
+  CHECK(f != nullptr) << "Require tvm_get_section_size to exist in registry";
+  size_t size = (*f)(binary_name, SectionToString(section));
+  return size;
 }
 
 std::unordered_map<std::string, void*> GetSymbolMap(std::string binary) {
-  return std::unordered_map<std::string, void*>();
+  const auto* f = Registry::Get("tvm_get_symbol_map");
+  CHECK(f != nullptr) << "Require tvm_get_symbol_map to exist in registry";
+  TVMByteArray arr;
+  arr.data = &binary[0];
+  arr.size = binary.length();
+  std::string map_str = (*f)(arr);
+  // parse symbols and addresses from returned string
+  std::unordered_map<std::string, void*> symbol_map;
+  std::stringstream stream;
+  stream << map_str;
+  std::string name;
+  void* addr;
+  stream >> name;
+  stream >> std::hex >> addr;
+  while (stream) {
+    symbol_map[name] = addr;
+    stream >> name;
+    stream >> std::hex >> addr;
+  }
+  return symbol_map;
 }
 }  // namespace runtime
 }  // namespace tvm

tests/python/contrib/test_binutil.py

@@ -0,0 +1,96 @@
+import tvm
+import subprocess
+from tvm.contrib import util
+from tvm.contrib import cc
+from tvm.contrib.binutil import *
+
+
+def make_binary():
+    prog = "int a = 7; \
+            int main() { \
+                int b = 5; \
+                return 0; \
+            }"
+    tmp_dir = util.tempdir()
+    tmp_source = tmp_dir.relpath("source.c")
+    tmp_obj = tmp_dir.relpath("obj.o")
+    with open(tmp_source, "w") as f:
+        f.write(prog)
+    p1 = subprocess.Popen(["gcc", "-c", tmp_source, "-o", tmp_obj],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.STDOUT)
+    p1.communicate()
+    prog_bin = bytearray(open(tmp_obj, "rb").read())
+    return prog_bin
+
+
+def test_tvm_get_section_size(binary):
+    tmp_dir = util.tempdir()
+    tmp_bin = tmp_dir.relpath("obj.bin")
+    with open(tmp_bin, "wb") as f:
+        f.write(binary)
+    def verify():
+        print("Text section size: %d" % tvm_get_section_size(tmp_bin, "text"))
+        print("Data section size: %d" % tvm_get_section_size(tmp_bin, "data"))
+        print("Bss section size: %d" % tvm_get_section_size(tmp_bin, "bss"))
+        print
+    verify()
+
+
+def test_tvm_relocate_binary(binary):
+    tmp_dir = util.tempdir()
+    tmp_bin = tmp_dir.relpath("obj.bin")
+    with open(tmp_bin, "wb") as f:
+        f.write(binary)
+    def verify():
+        rel_bin = tvm_relocate_binary(tmp_bin, "0x0", "0x10000", "0x20000")
+        print("Relocated binary section sizes")
+        test_tvm_get_section_size(rel_bin)
+        relf = tmp_dir.relpath("rel.bin")
+        with open(relf, "wb") as f:
+            f.write(rel_bin)
+        p1 = subprocess.Popen(["nm", "-C", "--defined-only", relf],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT)
+        (out, _) = p1.communicate()
+        print("Relocated binary symbols")
+        print(out)
+        print
+    verify()
+
+
+def test_tvm_read_binary_section(binary):
+    tmp_dir = util.tempdir()
+    tmp_bin = tmp_dir.relpath("obj.bin")
+    with open(tmp_bin, "wb") as f:
+        f.write(binary)
+    def verify():
+        text_bin = tvm_read_binary_section(tmp_bin, "text")
+        data_bin = tvm_read_binary_section(tmp_bin, "data")
+        bss_bin = tvm_read_binary_section(tmp_bin, "bss")
+        print("Read text section part of binary? %r" % (text_bin in binary))
+        print("Read data section part of binary? %r" % (data_bin in binary))
+        print("Read bss section part of binary? %r" % (bss_bin in binary))
+        print
+    verify()
+
+
+def test_tvm_get_symbol_map(binary):
+    tmp_dir = util.tempdir()
+    tmp_bin = tmp_dir.relpath("obj.bin")
+    with open(tmp_bin, "wb") as f:
+        f.write(binary)
+    def verify():
+        rel_bin = tvm_relocate_binary(tmp_bin, "0x0", "0x10000", "0x20000")
+        symbol_map = tvm_get_symbol_map(rel_bin)
+        print("Obtained symbol map")
+        print(symbol_map)
+    verify()
+
+
+if __name__ == "__main__":
+    prog_bin = make_binary()
+    test_tvm_get_section_size(prog_bin)
+    test_tvm_relocate_binary(prog_bin)
+    test_tvm_read_binary_section(prog_bin)
+    test_tvm_get_symbol_map(prog_bin)