[ETHOSN] Add support for Ethos-N 21.08 driver stack release. (apache#…

…9596)

Co-authored-by: Tristan O'Connor <tristan.oconnor@arm.com>

Co-authored-by: Tristan O'Connor <tristan.oconnor@arm.com>

cmake/utils/FindEthosN.cmake

@@ -61,19 +61,22 @@ macro(find_ethosn use_ethosn)
     set(ETHOSN_PACKAGE_VERSION "0.1.1")
     set(ETHOSN_DEFINITIONS -DETHOSN_API_VERSION=${USE_ETHOSN_API_VERSION})
 
-    if(${USE_ETHOSN_HW} MATCHES ${IS_TRUE_PATTERN})
-      # Runtime hardware support
-      find_path(_DL_DIR NAMES Network.hpp
-        PATHS ${__ethosn_stack}/include/ethosn_driver_library)
-      string(REGEX REPLACE "/ethosn_driver_library" "" _DL_DIR2 ${_DL_DIR})
-      list(APPEND ETHOSN_INCLUDE_DIRS "${_DL_DIR2}")
+    # Runtime hardware support. Driver library also needed for
+    # test support.
+    find_path(_DL_DIR NAMES Network.hpp
+      PATHS ${__ethosn_stack}/include/ethosn_driver_library)
+    string(REGEX REPLACE "/ethosn_driver_library" "" _DL_DIR2 ${_DL_DIR})
+    list(APPEND ETHOSN_INCLUDE_DIRS "${_DL_DIR2}")
 
-      find_library(ETHOSN_RUNTIME_LIBRARY NAMES EthosNDriver
-        PATHS ${__ethosn_stack}/lib)
-      find_library(ETHOSN_RUNTIME_LIBRARY NAMES EthosNDriver)
+    find_library(ETHOSN_RUNTIME_LIBRARY NAMES EthosNDriver
+      PATHS ${__ethosn_stack}/lib)
+    find_library(ETHOSN_RUNTIME_LIBRARY NAMES EthosNDriver)
+    if(${USE_ETHOSN_HW} MATCHES ${IS_TRUE_PATTERN})
       set(ETHOSN_DEFINITIONS -DETHOSN_HW -DETHOSN_API_VERSION=${USE_ETHOSN_API_VERSION})
-    endif ()
-
+    else()
+      set(ETHOSN_DEFINITIONS -DETHOSN_API_VERSION=${USE_ETHOSN_API_VERSION})
+    endif()
+
     if(ETHOSN_COMPILER_LIBRARY)
       set(ETHOSN_FOUND TRUE)
     endif()

docker/install/ubuntu_install_ethosn_driver_stack.sh

@@ -22,7 +22,7 @@ set -o pipefail
 
 repo_url="https://github.com/Arm-software/ethos-n-driver-stack"
 repo_dir="ethosn-driver"
-repo_revision="21.02"
+repo_revision="21.08"
 install_path="/opt/arm/$repo_dir"
 
 tmpdir=$(mktemp -d)

src/relay/backend/contrib/ethosn/codegen.cc

@@ -568,7 +568,7 @@ runtime::ethosn::OrderedCompiledNetwork EthosnCompiler::CompileEthosnFunc(const
   // the inputs/outputs from the TVM runtime to the inputs/outputs of the compiled network
   runtime::ethosn::OrderedCompiledNetwork ordered_network;
   ordered_network.name = gvar->name_hint;
-  ordered_network.cmm = std::move(compiled_network);
+  ordered_network.compiled_cmm = std::move(compiled_network);
   ordered_network.inputs = input_output_order.first;
   ordered_network.outputs = input_output_order.second;
   return ordered_network;

src/relay/backend/contrib/ethosn/ethosn_api_version.h

@@ -33,13 +33,14 @@
  * along with associated compatibility measures when no
  * longer necessary.
  */
-#if ETHOSN_SUPPORT_LIBRARY_VERSION_PATCH == 2
-#define _ETHOSN_API_VERSION_ 2102
+#if ETHOSN_SUPPORT_LIBRARY_VERSION_MAJOR == 1 && ETHOSN_SUPPORT_LIBRARY_VERSION_MINOR == 1 && \
+    ETHOSN_SUPPORT_LIBRARY_VERSION_PATCH == 0
+#define _ETHOSN_API_VERSION_ 2108
 #else
 #ifndef ETHOSN_API_VERSION
-#define _ETHOSN_API_VERSION_ 2011
+#define _ETHOSN_API_VERSION_ 2102
 #elif ~(~ETHOSN_API_VERSION + 0) == 0 && ~(~ETHOSN_API_VERSION + 1) == 1
-#define _ETHOSN_API_VERSION_ 2011
+#define _ETHOSN_API_VERSION_ 2102
 #else
 #define _ETHOSN_API_VERSION_ ETHOSN_API_VERSION
 #endif

src/runtime/contrib/ethosn/ethosn_device.cc

@@ -22,6 +22,8 @@
  * \brief Arm(R) Ethos(TM)-N NPU device integration.
  */
 
+#include "ethosn_device.h"
+
 #include <dlpack/dlpack.h>
 #include <poll.h>
 #include <tvm/runtime/ndarray.h>
@@ -32,6 +34,7 @@
 #include <memory>
 
 #include "ethosn_driver_library/Buffer.hpp"
+#include "ethosn_runtime.h"
 #include "ethosn_support_library/Support.hpp"
 
 #if defined ETHOSN_HW
@@ -43,7 +46,6 @@ namespace tvm {
 namespace runtime {
 namespace ethosn {
 
-namespace sl = ::ethosn::support_library;
 namespace dl = ::ethosn::driver_library;
 
 bool WaitForInference(dl::Inference* inference, int timeout) {
@@ -95,18 +97,25 @@ void CreateBuffers(std::vector<std::shared_ptr<dl::Buffer> >* fm,
   }
 }
 
+#if _ETHOSN_API_VERSION_ <= 2102
 bool Inference(tvm::runtime::TVMArgs args, sl::CompiledNetwork* network,
                const std::vector<uint32_t>& input_order,
                const std::vector<uint32_t>& output_order) {
+#else
+bool Inference(tvm::runtime::TVMArgs args, dl::Network* npu,
+               const std::vector<uint32_t>& input_order,
+               const std::vector<uint32_t>& output_order) {
+#endif
   // Unpack parameters
   uint8_t argc = 0;
-  std::vector<DLTensor*> inputs(input_order.size());
-  for (uint8_t i = 0; i < network->GetInputBufferInfos().size(); i++) {
+  size_t n_inputs = input_order.size();
+  size_t n_outputs = output_order.size();
+  std::vector<DLTensor*> inputs(n_inputs);
+  for (uint8_t i = 0; i < n_inputs; i++) {
     inputs[input_order[i]] = args[argc++];
   }
-  auto out_infos = network->GetOutputBufferInfos();
-  std::vector<DLTensor*> outputs(output_order.size());
-  for (uint8_t i = 0; i < network->GetOutputBufferInfos().size(); i++) {
+  std::vector<DLTensor*> outputs(n_outputs);
+  for (uint8_t i = 0; i < n_outputs; i++) {
     outputs[output_order[i]] = args[argc++];
   }
 
@@ -128,7 +137,9 @@ bool Inference(tvm::runtime::TVMArgs args, sl::CompiledNetwork* network,
     ofm_raw[i] = ofm[i].get();
   }
 
+#if _ETHOSN_API_VERSION_ <= 2102
   auto npu = std::make_unique<dl::Network>(*network);
+#endif
 
   // Execute the inference.
   std::unique_ptr<dl::Inference> result(
@@ -197,11 +208,17 @@ TVM_REGISTER_GLOBAL("relay.ethos-n.test.infra.inference_result")
     });
 
 // Allow the ethos-n support code to be tested without a device
+#if _ETHOSN_API_VERSION_ <= 2102
 bool Inference(tvm::runtime::TVMArgs args, sl::CompiledNetwork* network,
                const std::vector<uint32_t>& input_order,
                const std::vector<uint32_t>& output_order) {
+#else
+bool Inference(tvm::runtime::TVMArgs args, dl::Network* /* npu */,
+               const std::vector<uint32_t>& input_order,
+               const std::vector<uint32_t>& output_order) {
+#endif
   std::vector<DLTensor*> outputs;
-  for (int argc = network->GetInputBufferInfos().size(); argc < args.size(); argc++) {
+  for (int argc = input_order.size(); argc < args.size(); argc++) {
     outputs.push_back(args[argc]);
   }
   bool rc = false;

src/runtime/contrib/ethosn/ethosn_device.h

@@ -24,18 +24,28 @@
 #ifndef TVM_RUNTIME_CONTRIB_ETHOSN_ETHOSN_DEVICE_H_
 #define TVM_RUNTIME_CONTRIB_ETHOSN_ETHOSN_DEVICE_H_
 
+#include <tvm/runtime/registry.h>
+
 #include <vector>
 
-#include "ethosn_support_library/Support.hpp"
+#include "../../../relay/backend/contrib/ethosn/ethosn_api_version.h"
+#include "ethosn_runtime.h"
 
 namespace tvm {
 namespace runtime {
 namespace ethosn {
 
-namespace sl = ::ethosn::support_library;
+namespace dl = ::ethosn::driver_library;
+
+using tvm::runtime::TVMArgs;
 
-bool Inference(tvm::runtime::TVMArgs args, sl::CompiledNetwork* network,
-               const std::vector<uint32_t>& input_order, const std::vector<uint32_t>& output_order);
+#if _ETHOSN_API_VERSION_ <= 2102
+bool Inference(TVMArgs args, sl::CompiledNetwork* npu, const std::vector<uint32_t>& input_order,
+               const std::vector<uint32_t>& output_order);
+#else
+bool Inference(TVMArgs args, dl::Network* npu, const std::vector<uint32_t>& input_order,
+               const std::vector<uint32_t>& output_order);
+#endif
 
 }  // namespace ethosn
 }  // namespace runtime

src/runtime/contrib/ethosn/ethosn_runtime.cc

@@ -40,19 +40,24 @@
 #include "ethosn_device.h"
 #include "ethosn_driver_library/Inference.hpp"
 #include "ethosn_driver_library/Network.hpp"
-#include "ethosn_support_library/Support.hpp"
 
 namespace tvm {
 namespace runtime {
 namespace ethosn {
 
-namespace sl = ::ethosn::support_library;
 namespace dl = ::ethosn::driver_library;
 
 EthosnModule::EthosnModule(std::vector<OrderedCompiledNetwork>* cmms) {
   for (auto& it : *cmms) {
     network_map_[it.name].name = it.name;
-    network_map_[it.name].cmm = std::move(it.cmm);
+    if (it.compiled_cmm != nullptr) {
+      network_map_[it.name].compiled_cmm = std::move(it.compiled_cmm);
+    }
+#if _ETHOSN_API_VERSION_ > 2102
+    if (it.runtime_cmm != nullptr) {
+      network_map_[it.name].runtime_cmm = std::move(it.runtime_cmm);
+    }
+#endif
     network_map_[it.name].inputs = it.inputs;
     network_map_[it.name].outputs = it.outputs;
   }
@@ -62,8 +67,13 @@ PackedFunc EthosnModule::GetFunction(const std::string& name,
                                      const ObjectPtr<Object>& sptr_to_self) {
   if (network_map_.find(name) != network_map_.end()) {
     return PackedFunc([sptr_to_self, this, name](TVMArgs args, TVMRetValue* rv) {
-      *rv = Inference(args, network_map_[name].cmm.get(), network_map_[name].inputs,
+#if _ETHOSN_API_VERSION_ <= 2102
+      *rv = Inference(args, network_map_[name].compiled_cmm.get(), network_map_[name].inputs,
                       network_map_[name].outputs);
+#else
+      *rv = Inference(args, network_map_[name].runtime_cmm.get(), network_map_[name].inputs,
+                      network_map_[name].outputs);
+#endif
     });
   } else {
     return PackedFunc();
@@ -75,7 +85,8 @@ void EthosnModule::SaveToBinary(dmlc::Stream* stream) {
   for (const auto& it : network_map_) {
     stream->Write(it.first);
     std::stringstream ss;
-    it.second.cmm->Serialize(ss);
+    ICHECK(it.second.compiled_cmm != nullptr);
+    it.second.compiled_cmm->Serialize(ss);
     stream->Write(ss.str());
     stream->Write(it.second.inputs.size());
     stream->Write(&it.second.inputs[0], sizeof(uint32_t) * it.second.inputs.size());
@@ -102,7 +113,11 @@ Module EthosnModule::LoadFromBinary(void* strm) {
     // Read the serialized command stream
     stream->Read(&cmm);
     std::istringstream cmm_strm(cmm);
-    compiled.cmm = sl::DeserializeCompiledNetwork(cmm_strm);
+#if _ETHOSN_API_VERSION_ <= 2102
+    compiled.compiled_cmm = sl::DeserializeCompiledNetwork(cmm_strm);
+#else
+    compiled.runtime_cmm = std::make_unique<dl::Network>(cmm.c_str(), cmm.size());
+#endif
     // Read the number of inputs
     stream->Read<uint64_t>(&input_size);
     auto size = static_cast<size_t>(input_size);

src/runtime/contrib/ethosn/ethosn_runtime.h

@@ -28,20 +28,27 @@
 
 #include <map>
 #include <memory>
+#include <set>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
+#include "../../../relay/backend/contrib/ethosn/ethosn_api_version.h"
+#include "ethosn_driver_library/Network.hpp"
 #include "ethosn_support_library/Support.hpp"
 
 namespace tvm {
 namespace runtime {
 namespace ethosn {
 
 namespace sl = ::ethosn::support_library;
+namespace dl = ::ethosn::driver_library;
 
 struct OrderedCompiledNetwork {
-  std::unique_ptr<sl::CompiledNetwork> cmm;
+  std::unique_ptr<sl::CompiledNetwork> compiled_cmm;
+#if _ETHOSN_API_VERSION_ > 2102
+  std::unique_ptr<dl::Network> runtime_cmm;
+#endif
   std::string name;
   std::vector<uint32_t> inputs;
   std::vector<uint32_t> outputs;

tests/python/contrib/test_ethosn/test_conv2d.py

@@ -181,10 +181,7 @@ def test_conv2d():
 
 @requires_ethosn
 def test_conv2d_failure():
-    _scale_error_msg = (
-        "Overall scale (of the input * weights / output) should be in the range [0, 1)"
-    )
-
+    lb = "2.328306e-10" if tei.get_ethosn_api_version() > 2102 else "0"
     trials = [
         (
             (1, 4, 4, 4),
@@ -203,7 +200,7 @@ def test_conv2d_failure():
             "uint8",
             8,
             "HWIO",
-            _scale_error_msg,
+            f"Overall scale (of the input * weights / output) should be in the range [{lb}, 1)",
         ),
         (
             (1, 4, 4, 4),

tests/python/contrib/test_ethosn/test_fullyconnected.py

@@ -91,6 +91,7 @@ def test_fullyconnected():
 
 @requires_ethosn
 def test_fullyconnected_failure():
+    lb = "2.328306e-10" if tei.get_ethosn_api_version() > 2102 else "0"
     trials = [
         (
             (1, 64),
@@ -102,7 +103,7 @@ def test_fullyconnected_failure():
             0,
             1,
             "uint8",
-            "Overall scale (of the input * weights / output) should be in the range [0, 1)",
+            f"Overall scale (of the input * weights / output) should be in the range [{lb}, 1)",
         ),
         (
             (1, 1, 1, 64),

tests/python/contrib/test_ethosn/test_networks.py

@@ -125,11 +125,11 @@ def test_mobilenet_v1():
     # on hardware that isn't available in CI.
     _compile_hash = {"0433d3c3947a067b36f0228bdb5f1838"}
     if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-        _compile_hash = {"e4ed29dceb1187505948ab17fc3cc6d6"}
-    if tei.get_ethosn_api_version() == 2011:
-        _compile_hash = {"9c9f63b30824f5b223cdb27d2f22c857"}
+        _compile_hash = {"393a19dfb980345cdd3bbeddbc36424d"}
+    if tei.get_ethosn_api_version() == 2102:
+        _compile_hash = {"46ccafc840633633aca441645e41b444"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-            _compile_hash = {"cd13279061df2319124a7aac81581d81"}
+            _compile_hash = {"e4ed29dceb1187505948ab17fc3cc6d6"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
@@ -150,13 +150,13 @@ def test_inception_v3():
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    _compile_hash = {"43dc2097127eb224c0191b1a15f8acca"}
+    _compile_hash = {"46ccafc840633633aca441645e41b444"}
     if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-        _compile_hash = {"7db23387bdc5af6eaa1ae3f7d456caf0"}
-    if tei.get_ethosn_api_version() == 2011:
-        _compile_hash = {"46ccafc840633633aca441645e41b444"}
+        _compile_hash = {"2c7ff5487e1a21e62b3b42eec624fed4"}
+    if tei.get_ethosn_api_version() == 2102:
+        _compile_hash = {"43dc2097127eb224c0191b1a15f8acca"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-            _compile_hash = {"4a33f397ac3e15c0f9869f7b8286fc2f"}
+            _compile_hash = {"7db23387bdc5af6eaa1ae3f7d456caf0"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite_11_05_08/inception_v3_quant.tgz",
@@ -178,11 +178,11 @@ def test_inception_v4():
     # on hardware that isn't available in CI.
     _compile_hash = {"fab6c2297502f95d33079c6ce1a737f9"}
     if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-        _compile_hash = {"8da68849b75613ac3dffd3fff2dd87da"}
-    if tei.get_ethosn_api_version() == 2011:
-        _compile_hash = {"4a1a56393078367dd27915a188d6a6af"}
+        _compile_hash = {"4245dbd02e1432dc261a67fc8e632a00"}
+    if tei.get_ethosn_api_version() == 2102:
+        _compile_hash = {"fab6c2297502f95d33079c6ce1a737f9"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-            _compile_hash = {"905caf389dd6b868aeff6acbca1fecef"}
+            _compile_hash = {"8da68849b75613ac3dffd3fff2dd87da"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/inception_v4_299_quant_20181026.tgz",
@@ -204,11 +204,11 @@ def test_ssd_mobilenet_v1():
     # on hardware that isn't available in CI.
     _compile_hash = {"2345cf5d6c0013bad7c76dcccee9d862", "7795b6c67178da9d1f9b98063bad75b1"}
     if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-        _compile_hash = {"928dc6ae5ce49a4ad63ca87f7575970f", "b092f9820f7e9341fc53daa781b98772"}
-    if tei.get_ethosn_api_version() == 2011:
-        _compile_hash = {"10826406ae724e52f360a06c35ced09d", "9a484d5ecec7acb18c9d6bc6058be031"}
+        _compile_hash = {"5ee8ed6af9a7f31fc14957b51a8e7423", "e6a91ccc47ba4c6b4614fcd676bd726f"}
+    if tei.get_ethosn_api_version() == 2102:
+        _compile_hash = {"7795b6c67178da9d1f9b98063bad75b1", "10826406ae724e52f360a06c35ced09d"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-            _compile_hash = {"425b38830f34b6eb448fa77dbfe9ac96", "de49128643cbf1c659a9a63aad1cba62"}
+            _compile_hash = {"928dc6ae5ce49a4ad63ca87f7575970f", "b092f9820f7e9341fc53daa781b98772"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip",