[WASMFS] JS File Backend Structure (#15562)

Relevant Issue: #15041

Tentative JS File Backend Structure

Introduced user-visible header file.
Users can create a new JS Backend and create JS Backed Files under both In-Memory and JS Backend directories.

Author: ethanalee

src/library_wasmfs.js

@@ -1,7 +1,9 @@
 var WasmfsLibrary = {
+  $wasmFS$JSMemoryFiles : [],
+  $wasmFS$JSMemoryFreeList: [],
   $wasmFS$preloadedFiles: [],
   $wasmFS$preloadedDirs: [],
-  $FS__deps: ['$wasmFS$preloadedFiles', '$wasmFS$preloadedDirs'],
+  $FS__deps: ['$wasmFS$preloadedFiles', '$wasmFS$preloadedDirs', '$wasmFS$JSMemoryFiles', '$wasmFS$JSMemoryFreeList'],
   $FS : {
     // TODO: Clean up the following functions - currently copied from library_fs.js directly.
     createPreloadedFile: function(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) {
@@ -85,6 +87,54 @@ var WasmfsLibrary = {
       // For file preloading, cwd should be '/' to begin with.
       return '/';
     }
+  },
+  _emscripten_write_js_file: function(index, buffer, length, offset) {
+    try {
+      if (!wasmFS$JSMemoryFiles[index]) {
+        // Initialize typed array on first write operation.
+        wasmFS$JSMemoryFiles[index] = new Uint8Array(offset + length);
+      }
+      
+      if (offset + length > wasmFS$JSMemoryFiles[index].length) {
+        // Resize the typed array if the length of the write buffer exceeds its capacity.
+        var oldContents = wasmFS$JSMemoryFiles[index];
+        var newContents = new Uint8Array(offset + length);
+        newContents.set(oldContents);
+        wasmFS$JSMemoryFiles[index] = newContents;
+      }
+
+      wasmFS$JSMemoryFiles[index].set(HEAPU8.subarray(buffer, buffer + length), offset);
+      return 0;
+    } catch (err) {
+      return {{{ cDefine('EIO') }}};
+    }
+  },
+  _emscripten_read_js_file: function(index, buffer, length, offset) {
+    try {
+      HEAPU8.set(wasmFS$JSMemoryFiles[index].subarray(offset, offset + length), buffer);
+      return 0;
+    } catch (err) {
+      return {{{ cDefine('EIO') }}};
+    }
+  },
+  _emscripten_get_js_file_size: function(index) {
+    return wasmFS$JSMemoryFiles[index] ? wasmFS$JSMemoryFiles[index].length : 0;
+  },
+  _emscripten_create_js_file: function() {
+    // Find a free entry in the $wasmFS$JSMemoryFreeList or append a new entry to
+    // wasmFS$JSMemoryFiles.
+    if (wasmFS$JSMemoryFreeList.length) {
+      // Pop off the top of the free list.
+      var index = wasmFS$JSMemoryFreeList.pop();
+      return index;
+    }
+    wasmFS$JSMemoryFiles.push(null);
+    return wasmFS$JSMemoryFiles.length - 1;
+  },
+  _emscripten_remove_js_file: function(index) {
+    wasmFS$JSMemoryFiles[index] = null;
+    // Add the index to the free list.
+    wasmFS$JSMemoryFreeList.push(index);
   }
 }
 

system/include/emscripten/wasmfs.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2021 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <sys/stat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct Backend* backend_t;
+
+// Obtains the backend_t of a specified path.
+backend_t wasmfs_get_backend_by_path(char* path);
+
+// Obtains the backend_t of a specified fd.
+backend_t wasmfs_get_backend_by_fd(int fd);
+
+// Creates a JSFile Backend in the new file system.
+backend_t wasmfs_create_js_file_backend();
+
+// Creates a file in a specific backend and returns an fd to an open file.
+uint32_t wasmfs_create_file(char* pathname, mode_t mode, backend_t backend);
+
+// Creates a new directory in the new file system under a specific backend.
+long wasmfs_create_directory(char* path, long mode, backend_t backend);
+
+#ifdef __cplusplus
+}
+#endif

system/lib/wasmfs/backend.h

@@ -14,8 +14,8 @@
 
 namespace wasmfs {
 // A backend (or modular backend) provides a base for the new file system to
-// extend its storage capabilities. Files and directories will be represented in
-// the file system structure, but their underlying backing could exist in
+// extend its storage capabilities. Files and directories will be represented
+// in the file system structure, but their underlying backing could exist in
 // persistent storage, another thread, etc.
 class Backend {
 
@@ -29,7 +29,5 @@ class Backend {
 // Note: Backends will be defined in cpp files, but functions to instantiate
 // them will be defined in a header file. This is so that any unused backends
 // are not linked in if they are not called.
-// TODO: In the next PR, a user-visible header is introduced. Update this
-// comment then.
 backend_t createMemoryFileBackend();
 } // namespace wasmfs

system/lib/wasmfs/file.h

@@ -2,6 +2,7 @@
 // Emscripten is available under two separate licenses, the MIT license and the
 // University of Illinois/NCSA Open Source License.  Both these licenses can be
 // found in the LICENSE file.
+
 // This file defines the file object of the new file system.
 // Current Status: Work in Progress.
 // See https://github.com/emscripten-core/emscripten/issues/15041.
@@ -25,6 +26,7 @@ class Backend;
 // This represents an opaque pointer to a Backend. A user may use this to
 // specify a backend in file operations.
 using backend_t = Backend*;
+const backend_t NullBackend = nullptr;
 
 class File : public std::enable_shared_from_this<File> {
 
@@ -62,6 +64,8 @@ class File : public std::enable_shared_from_this<File> {
     return (ino_t)this;
   }
 
+  backend_t getBackend() { return backend; }
+
   class Handle {
 
   protected:
@@ -101,7 +105,8 @@ class File : public std::enable_shared_from_this<File> {
   }
 
 protected:
-  File(FileKind kind, mode_t mode) : kind(kind), mode(mode) {}
+  File(FileKind kind, mode_t mode, backend_t backend)
+    : kind(kind), mode(mode), backend(backend) {}
   // A mutex is needed for multiple accesses to the same file.
   std::recursive_mutex mutex;
 
@@ -121,6 +126,9 @@ class File : public std::enable_shared_from_this<File> {
   // dependencies where the parent and child have shared_ptrs that reference
   // each other. This prevents the case in which an uncollectable cycle occurs.
   std::weak_ptr<File> parent;
+
+  // This specifies which backend a file is associated with.
+  backend_t backend;
 };
 
 class DataFile : public File {
@@ -131,7 +139,8 @@ class DataFile : public File {
 
 public:
   static constexpr FileKind expectedKind = File::DataFileKind;
-  DataFile(mode_t mode) : File(File::DataFileKind, mode) {}
+  DataFile(mode_t mode, backend_t backend)
+    : File(File::DataFileKind, mode, backend) {}
   virtual ~DataFile() = default;
 
   class Handle : public File::Handle {
@@ -161,15 +170,10 @@ class Directory : public File {
   // This value was also copied from the existing file system.
   size_t getSize() override { return 4096; }
 
-  // This specifies which backend a directory is associated with. By default,
-  // files and sub-directories added to this directory's entries will be created
-  // through this same backend unless an alternative is specified.
-  backend_t backend;
-
 public:
   static constexpr FileKind expectedKind = File::DirectoryKind;
   Directory(mode_t mode, backend_t backend)
-    : File(File::DirectoryKind, mode), backend(backend) {}
+    : File(File::DirectoryKind, mode, backend) {}
 
   struct Entry {
     std::string name;
@@ -231,8 +235,6 @@ class Directory : public File {
       return entries;
     }
 
-    backend_t getBackend() { return getDir()->backend; }
-
 #ifdef WASMFS_DEBUG
     void printKeys() {
       for (auto keyPair : getDir()->entries) {
@@ -253,7 +255,6 @@ class Directory : public File {
     }
   }
 };
-
 // Obtains parent directory of a given pathname.
 // Will return a nullptr if the parent is not a directory.
 // Will error if the forbiddenAncestor is encountered while processing.
@@ -265,8 +266,8 @@ getDir(std::vector<std::string>::iterator begin,
        long& err,
        std::shared_ptr<File> forbiddenAncestor = nullptr);
 
-// Return a vector of the '/'-delimited components of a path. The first element
-// will be "/" iff the path is an absolute path.
+// Return a vector of the '/'-delimited components of a path. The first
+// element will be "/" iff the path is an absolute path.
 std::vector<std::string> splitPath(char* pathname);
 
 } // namespace wasmfs

system/lib/wasmfs/js_file_backend.cpp

@@ -0,0 +1,78 @@
+// Copyright 2021 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+// This file defines the JS file backend and JS file of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#include "backend.h"
+#include "wasmfs.h"
+
+using js_index_t = uint32_t;
+
+extern "C" {
+int _emscripten_write_js_file(js_index_t index,
+                              const uint8_t* buffer,
+                              size_t length,
+                              off_t offset);
+int _emscripten_read_js_file(js_index_t index,
+                             const uint8_t* buffer,
+                             size_t length,
+                             off_t offset);
+int _emscripten_get_js_file_size(js_index_t index);
+int _emscripten_create_js_file();
+void _emscripten_remove_js_file(js_index_t index);
+}
+
+namespace wasmfs {
+
+// This class describes a file that lives in JS Memory
+class JSFile : public DataFile {
+  // This index indicates the location of the JS File in the backing JS array.
+  js_index_t index;
+
+  // JSFiles will write from a Wasm Memory buffer into the backing JS array.
+  __wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
+    return _emscripten_write_js_file(index, buf, len, offset);
+  }
+
+  // JSFiles will read from the backing JS array into a Wasm Memory buffer.
+  __wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
+    // The caller should have already checked that the offset + len does
+    // not exceed the file's size.
+    assert(offset + len <= getSize());
+    return _emscripten_read_js_file(index, buf, len, offset);
+  }
+
+  // The size of the JSFile is defined as the length of the backing JS array.
+  size_t getSize() override { return _emscripten_get_js_file_size(index); }
+
+public:
+  JSFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
+    // Create a new file in the backing JS array and store its index.
+    index = _emscripten_create_js_file();
+  }
+
+  // Remove the typed array file contents in the backing JS array.
+  ~JSFile() { _emscripten_remove_js_file(index); }
+};
+
+class JSFileBackend : public Backend {
+
+public:
+  std::shared_ptr<DataFile> createFile(mode_t mode) override {
+    return std::make_shared<JSFile>(mode, this);
+  }
+  std::shared_ptr<Directory> createDirectory(mode_t mode) override {
+    return std::make_shared<Directory>(mode, this);
+  }
+};
+
+// This function is exposed to users to instantiate a new JSBackend.
+extern "C" backend_t wasmfs_create_js_file_backend() {
+  return wasmFS.addBackend(std::make_unique<JSFileBackend>());
+}
+
+} // namespace wasmfs

system/lib/wasmfs/memory_file.h

@@ -2,6 +2,7 @@
 // Emscripten is available under two separate licenses, the MIT license and the
 // University of Illinois/NCSA Open Source License.  Both these licenses can be
 // found in the LICENSE file.
+
 // This file defines the memory file class of the new file system.
 // This should be the only backend file type defined in a header since it is the
 // default type. Current Status: Work in Progress. See
@@ -23,7 +24,7 @@ class MemoryFile : public DataFile {
   size_t getSize() override { return buffer.size(); }
 
 public:
-  MemoryFile(mode_t mode) : DataFile(mode) {}
+  MemoryFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {}
 
   class Handle : public DataFile::Handle {
 

system/lib/wasmfs/memory_file_backend.cpp

@@ -2,6 +2,7 @@
 // Emscripten is available under two separate licenses, the MIT license and the
 // University of Illinois/NCSA Open Source License.  Both these licenses can be
 // found in the LICENSE file.
+
 // This file defines the memory file backend of the new file system.
 // Current Status: Work in Progress.
 // See https://github.com/emscripten-core/emscripten/issues/15041.
@@ -15,7 +16,7 @@ class MemoryFileBackend : public Backend {
 
 public:
   std::shared_ptr<DataFile> createFile(mode_t mode) override {
-    return std::make_shared<MemoryFile>(mode);
+    return std::make_shared<MemoryFile>(mode, this);
   }
   std::shared_ptr<Directory> createDirectory(mode_t mode) override {
     return std::make_shared<Directory>(mode, this);

system/lib/wasmfs/streams.h

@@ -28,7 +28,7 @@ class StdinFile : public DataFile {
   size_t getSize() override { return 0; }
 
 public:
-  StdinFile(mode_t mode) : DataFile(mode) {}
+  StdinFile(mode_t mode) : DataFile(mode, NullBackend) {}
   static std::shared_ptr<StdinFile> getSingleton();
 };
 
@@ -45,7 +45,7 @@ class StdoutFile : public DataFile {
   size_t getSize() override { return 0; }
 
 public:
-  StdoutFile(mode_t mode) : DataFile(mode) {}
+  StdoutFile(mode_t mode) : DataFile(mode, NullBackend) {}
   static std::shared_ptr<StdoutFile> getSingleton();
 };
 
@@ -65,7 +65,7 @@ class StderrFile : public DataFile {
   size_t getSize() override { return 0; }
 
 public:
-  StderrFile(mode_t mode) : DataFile(mode) {}
+  StderrFile(mode_t mode) : DataFile(mode, NullBackend) {}
   static std::shared_ptr<StderrFile> getSingleton();
 };
 } // namespace wasmfs