introduce file_data_sink (#8921)

Summary:

This diff introduces a new data sink called `file_data_sink` to the Executorch runtime to have a data logging pipeline into file.

Differential Revision: D70541550

Author: Gasoonjia

devtools/etdump/data_sinks/file_data_sink.cpp

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <executorch/devtools/etdump/data_sinks/file_data_sink.h>
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
+#include <cstdio> // For FILE operations
+
+using ::executorch::runtime::Error;
+using ::executorch::runtime::Result;
+
+namespace executorch {
+namespace etdump {
+
+FileDataSink::FileDataSink(FileDataSink&& other) noexcept
+    : file_(other.file_), total_written_bytes_(other.total_written_bytes_) {
+  other.file_ = nullptr;
+}
+
+Result<FileDataSink> FileDataSink::create(const char* file_path) {
+  // Open the file and get the file pointer
+  FILE* file = fopen(file_path, "wb");
+  if (!file) {
+    // Return an error if the file cannot be accessed or created
+    return Error::AccessFailed;
+  }
+
+  // Return the successfully created FileDataSink
+  return FileDataSink(file);
+}
+
+FileDataSink::~FileDataSink() {
+  // Close the file
+  close();
+}
+
+Result<size_t> FileDataSink::write(const void* ptr, size_t size) {
+  if (!file_) {
+    ET_LOG(Error, "File not open, unable to write.");
+    return Error::AccessFailed;
+  }
+
+  size_t offset = total_written_bytes_;
+
+  if (size == 0) {
+    // No data to write, return current offset
+    return offset;
+  }
+
+  size_t written = fwrite(ptr, 1, size, file_);
+  if (written != size) {
+    ET_LOG(Error, "Write failed: wrote %zu bytes of %zu", written, size);
+    return Error::Internal;
+  }
+
+  total_written_bytes_ += written;
+  return offset;
+}
+
+size_t FileDataSink::get_used_bytes() const {
+  return total_written_bytes_;
+}
+
+void FileDataSink::close() {
+  if (file_) {
+    fclose(file_);
+    file_ = nullptr;
+  }
+}
+
+} // namespace etdump
+} // namespace executorch

devtools/etdump/data_sinks/file_data_sink.h

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <executorch/devtools/etdump/data_sinks/data_sink_base.h>
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
+#include <cstdio> // For FILE operations
+
+namespace executorch {
+namespace etdump {
+
+/**
+ * FileDataSink is a concrete implementation of the DataSinkBase class,
+ * designed to facilitate the direct writing of data to a file. It is
+ * particularly useful for scenarios where immediate data storage is
+ * required, such as logging or streaming data to a file for real-time
+ * analysis. The class manages file operations, including opening, writing,
+ * and closing the file, while handling potential errors during these
+ * operations.
+ */
+
+class FileDataSink : public DataSinkBase {
+ public:
+  /**
+   * Creates a FileDataSink with a given file path.
+   *
+   * @param[in] file_path The path to the file for writing data.
+   * @return A Result object containing either:
+   *         - A FileDataSink object if succees, or
+   *         - AccessFailed Error when the file cannot be accessed or created
+   */
+  static ::executorch::runtime::Result<FileDataSink> create(
+      const char* file_path);
+
+  /**
+   * Destructor that closes the file.
+   */
+  ~FileDataSink() override;
+
+  // Delete copy constructor and copy assignment operator
+  FileDataSink(const FileDataSink&) = delete;
+  FileDataSink& operator=(const FileDataSink&) = delete;
+
+  FileDataSink(FileDataSink&& other) noexcept;
+  FileDataSink& operator=(FileDataSink&& other) = default;
+
+  /**
+   * Writes data directly to the file.
+   *
+   * This function does not perform any alignment, and will overwrite
+   * any existing data in the file.
+   *
+   * @param[in] ptr A pointer to the data to be written into the file.
+   * @param[in] size The size of the data in bytes.
+   * @return A Result object containing either:
+   *         - The offset of the starting location of the data within the
+   *           file, or
+   *         - AccessFailedError if the file has been closed.
+   *         - InternalError if the os write operation fails.
+   */
+  ::executorch::runtime::Result<size_t> write(const void* ptr, size_t size)
+      override;
+
+  /**
+   * Gets the number of bytes currently written to the file.
+   *
+   * @return The amount of data currently stored in bytes.
+   */
+  size_t get_used_bytes() const override;
+
+  /**
+   * Closes the file, if it is open.
+   */
+  void close();
+
+ private:
+  /**
+   * Constructs a FileDataSink with a given file pointer.
+   *
+   * @param[in] file A valid file pointer for writing data.
+   */
+  explicit FileDataSink(FILE* file) : file_(file), total_written_bytes_(0) {}
+
+  FILE* file_;
+  size_t total_written_bytes_;
+};
+
+} // namespace etdump
+} // namespace executorch

devtools/etdump/data_sinks/targets.bzl

@@ -47,3 +47,4 @@ def define_common_targets():
         )
 
         define_data_sink_target("buffer_data_sink", aten_suffix)
+        define_data_sink_target("file_data_sink", aten_suffix)

devtools/etdump/data_sinks/tests/file_data_sink_test.cpp

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <executorch/devtools/etdump/data_sinks/file_data_sink.h>
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
+#include <executorch/runtime/platform/runtime.h>
+#include <gtest/gtest.h>
+#include <stdio.h> // tmpnam(), remove()
+#include <fstream>
+
+using namespace ::testing;
+using ::executorch::etdump::FileDataSink;
+using ::executorch::runtime::Error;
+using ::executorch::runtime::Result;
+
+class FileDataSinkTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    // Initialize the runtime environment
+    torch::executor::runtime_init();
+
+    // Define the file path for testing
+    std::array<char, L_tmpnam> buf;
+    const char* ret = std::tmpnam(buf.data());
+    ASSERT_NE(ret, nullptr) << "Could not generate temp file";
+    buf[L_tmpnam - 1] = '\0';
+    file_path_ = std::string(buf.data()) + "-executorch-testing";
+  }
+
+  void TearDown() override {
+    // Remove the test file
+    std::remove(file_path_.c_str());
+  }
+
+  std::string file_path_;
+};
+
+TEST_F(FileDataSinkTest, CreationExpectFail) {
+  // Create a FileDataSink instance with a valid file path
+  Result<FileDataSink> success = FileDataSink::create(file_path_.c_str());
+  ASSERT_TRUE(success.ok());
+
+  // Try to create another FileDataSink instance with an invalid file path
+  Result<FileDataSink> fail_with_invalid_file_path = FileDataSink::create("");
+  ASSERT_EQ(fail_with_invalid_file_path.error(), Error::AccessFailed);
+}
+
+TEST_F(FileDataSinkTest, WriteDataToFile) {
+  const char* data = "Hello, World!";
+  size_t data_size = strlen(data);
+
+  // Create a FileDataSink instance
+  Result<FileDataSink> result = FileDataSink::create(file_path_.c_str());
+  ASSERT_TRUE(result.ok());
+
+  FileDataSink* data_sink = &result.get();
+
+  // Write data to the file
+  Result<size_t> write_result = data_sink->write(data, data_size);
+  ASSERT_TRUE(write_result.ok());
+
+  size_t used_bytes = data_sink->get_used_bytes();
+  EXPECT_EQ(used_bytes, data_size);
+
+  data_sink->close();
+
+  // Expect fail if write again after close
+  Result<size_t> write_result_after_close = data_sink->write(data, data_size);
+  ASSERT_EQ(write_result_after_close.error(), Error::AccessFailed);
+
+  // Verify the file contents
+  std::ifstream file(file_path_, std::ios::binary);
+  file.seekg(0, std::ios::end);
+  size_t file_size = file.tellg();
+  file.seekg(0, std::ios::beg);
+  EXPECT_EQ(file_size, used_bytes);
+
+  // Read the file content and verify it matches the original data
+  std::vector<char> file_content(file_size);
+  file.read(file_content.data(), file_size);
+  file.close();
+
+  EXPECT_EQ(std::memcmp(file_content.data(), data, data_size), 0);
+}
+
+TEST_F(FileDataSinkTest, WriteMultipleDataAndCheckOffsets) {
+  const char* data1 = "Accelerate";
+  const char* data2 = "Core";
+  const char* data3 = "Experience";
+  size_t data1_size = strlen(data1);
+  size_t data2_size = strlen(data2);
+  size_t data3_size = strlen(data3);
+
+  // Create a FileDataSink instance
+  Result<FileDataSink> result = FileDataSink::create(file_path_.c_str());
+  ASSERT_TRUE(result.ok());
+  FileDataSink* data_sink = &result.get();
+
+  // Write multiple data chunks and check offsets
+  Result<size_t> offset1 = data_sink->write(data1, data1_size);
+  ASSERT_TRUE(offset1.ok());
+  EXPECT_EQ(offset1.get(), 0);
+
+  Result<size_t> offset2 = data_sink->write(data2, data2_size);
+  ASSERT_TRUE(offset2.ok());
+  EXPECT_EQ(offset2.get(), data1_size);
+
+  Result<size_t> offset3 = data_sink->write(data3, data3_size);
+  ASSERT_TRUE(offset3.ok());
+  EXPECT_EQ(offset3.get(), data1_size + data2_size + data3_size);
+  size_t used_bytes = data_sink->get_used_bytes();
+  EXPECT_EQ(used_bytes, data1_size + data2_size);
+
+  data_sink->close();
+
+  // Verify the file contents
+  std::ifstream file(file_path_, std::ios::binary);
+  file.seekg(0, std::ios::end);
+  size_t file_size = file.tellg();
+  file.seekg(0, std::ios::beg);
+  EXPECT_EQ(file_size, used_bytes);
+
+  // Read the file content
+  std::vector<char> file_content(file_size);
+  file.read(file_content.data(), file_size);
+  file.close();
+
+  // Verify each data chunk in the file using offsets
+  EXPECT_EQ(
+      std::memcmp(file_content.data() + offset1.get(), data1, data1_size), 0);
+  EXPECT_EQ(
+      std::memcmp(file_content.data() + offset2.get(), data2, data2_size), 0);
+  EXPECT_EQ(
+      std::memcmp(file_content.data() + offset3.get(), data3, data3_size), 0);
+}

devtools/etdump/data_sinks/tests/targets.bzl

@@ -1,20 +1,23 @@
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
 
-def define_common_targets():
-    """Defines targets that should be shared between fbcode and xplat.
-
-    The directory containing this targets.bzl file should also contain both
-    TARGETS and BUCK files that call this function.
-    """
-
-
+def define_data_sink_test(data_sink_name):
     runtime.cxx_test(
-        name = "buffer_data_sink_test",
+        name = data_sink_name + "_test",
         srcs = [
-            "buffer_data_sink_test.cpp",
+            data_sink_name + "_test.cpp",
         ],
         deps = [
-            "//executorch/devtools/etdump/data_sinks:buffer_data_sink",
+            "//executorch/devtools/etdump/data_sinks:" + data_sink_name,
             "//executorch/runtime/core/exec_aten/testing_util:tensor_util",
         ],
     )
+
+def define_common_targets():
+    """Defines targets that should be shared between fbcode and xplat.
+
+    The directory containing this targets.bzl file should also contain both
+    TARGETS and BUCK files that call this function.
+    """
+
+    define_data_sink_test("buffer_data_sink")
+    define_data_sink_test("file_data_sink")