diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc
index 491e1e49f805c..32414c607df6f 100644
--- a/src/runtime/pipeline/pipeline_executor.cc
+++ b/src/runtime/pipeline/pipeline_executor.cc
@@ -127,11 +127,11 @@ void PipelineExecutor::Init(const std::vector<Module>& modules, const std::strin
   // Use JSONReader to load pipeline configuration.
   std::istringstream is(pipeline_json);
   dmlc::JSONReader reader(&is);
-  ConfigPipelineExecution& pipeline_config = this->LoadConfig(&reader);
-  ICHECK(!pipeline_config.Empty()) << "The pipeline config information is empty.";
+  this->LoadConfig(&reader);
+  ICHECK(!pipeline_config_.Empty()) << "The pipeline config information is empty.";
   // Initialize the pipeline function class used for pipeline thread pool management
   // and schedule etc. This function returns the number of output.
-  num_outputs_ = pipeline_scheduler_.PipelineInit(modules, pipeline_config);
+  num_outputs_ = pipeline_scheduler_.PipelineInit(modules, pipeline_config_);
   return;
 }
 
diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h
index 28234e1bdcf08..1ae52e07c2607 100644
--- a/src/runtime/pipeline/pipeline_executor.h
+++ b/src/runtime/pipeline/pipeline_executor.h
@@ -131,7 +131,7 @@ class TVM_DLL PipelineExecutor : public ModuleNode {
   /*!\brief How many outputs are in this pipeline executor.*/
   size_t num_outputs_ = 0;
   /*!\brief Json loader.*/
-  ConfigPipelineExecution& LoadConfig(dmlc::JSONReader* reader) {
+  void LoadConfig(dmlc::JSONReader* reader) {
     reader->BeginObject();
     std::string key;
     while (reader->NextObjectItem(&key)) {
@@ -143,7 +143,7 @@ class TVM_DLL PipelineExecutor : public ModuleNode {
         LOG(FATAL) << "do not support key " << key;
       }
     }
-    return pipeline_config_;
+    return;
   }
 };
 }  // namespace runtime