Import fixes (#902)

* Apply clang-format to cc files

* Modify binaryproto namespace

* Add more required includes

* Add proto includes

* Assert message parsing succeeds in test

* Add optional keyword to proto fields to track presence. TESTS BROKEN.

* Update golden test data

Source/santad/BUILD

@@ -1124,7 +1124,7 @@ test_suite(
         ":SNTExecutionControllerTest",
         ":SNTRuleTableTest",
         ":SantadTest",
-        "//Source/santad/Logs/EndpointSecurity/Writers/FSSpool:fsspool_test"
+        "//Source/santad/Logs/EndpointSecurity/Writers/FSSpool:fsspool_test",
     ],
     visibility = ["//:santa_package_group"],
 )

Source/santad/Logs/EndpointSecurity/Serializers/Protobuf.mm

@@ -35,6 +35,7 @@
 #include "Source/santad/EventProviders/EndpointSecurity/EndpointSecurityAPI.h"
 #include "Source/santad/Logs/EndpointSecurity/Serializers/Utilities.h"
 #import "Source/santad/SNTDecisionCache.h"
+#include "google/protobuf/timestamp.pb.h"
 
 using google::protobuf::Arena;
 using google::protobuf::Timestamp;

Source/santad/Logs/EndpointSecurity/Serializers/ProtobufTest.mm

@@ -41,19 +41,13 @@
 #include "Source/santad/Logs/EndpointSecurity/Serializers/Protobuf.h"
 #include "Source/santad/Logs/EndpointSecurity/Serializers/Serializer.h"
 #import "Source/santad/SNTDecisionCache.h"
+#include "google/protobuf/any.pb.h"
+#include "google/protobuf/timestamp.pb.h"
 
 using google::protobuf::Timestamp;
 using google::protobuf::util::JsonPrintOptions;
-using santa::santad::event_providers::endpoint_security::EnrichedClose;
 using santa::santad::event_providers::endpoint_security::EnrichedEventType;
-using santa::santad::event_providers::endpoint_security::EnrichedExchange;
-using santa::santad::event_providers::endpoint_security::EnrichedExec;
-using santa::santad::event_providers::endpoint_security::EnrichedExit;
-using santa::santad::event_providers::endpoint_security::EnrichedFork;
-using santa::santad::event_providers::endpoint_security::EnrichedLink;
 using santa::santad::event_providers::endpoint_security::EnrichedMessage;
-using santa::santad::event_providers::endpoint_security::EnrichedRename;
-using santa::santad::event_providers::endpoint_security::EnrichedUnlink;
 using santa::santad::event_providers::endpoint_security::Enricher;
 using santa::santad::event_providers::endpoint_security::Message;
 using santa::santad::logs::endpoint_security::serializers::Protobuf;
@@ -168,7 +162,7 @@ void CheckSantaMessage(const ::pbv1::SantaMessage &santaMsg, const es_message_t
   const google::protobuf::Message &message = SantaMessageEvent(santaMsg);
 
   std::string json;
-  google::protobuf::util::MessageToJsonString(message, &json, options);
+  XCTAssertTrue(google::protobuf::util::MessageToJsonString(message, &json, options).ok());
   return json;
 }
 

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/binaryproto.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package binaryproto;
+package santa.fsspool.binaryproto;
 
 import "google/protobuf/any.proto";
 

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.cc

@@ -21,6 +21,7 @@
 #include <limits>
 #include <string>
 
+#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h"
 #include "absl/random/random.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
@@ -29,7 +30,6 @@
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "absl/time/time.h"
-#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h"
 
 namespace fsspool {
 
@@ -108,7 +108,9 @@ absl::Status WriteTmpFile(const std::string& path, absl::string_view msg) {
     if (Unlink(path.c_str()) < 0) {
       // This is very unlikely (e.g. somehow permissions for the file changed
       // since creation?), still worth logging the error.
-      return absl::ErrnoToStatus(errno, absl::StrCat("Writing to ", path, " failed (and deleting failed too)"));
+      return absl::ErrnoToStatus(
+          errno, absl::StrCat("Writing to ", path,
+                              " failed (and deleting failed too)"));
     }
     return write_status;
   }
@@ -172,18 +174,18 @@ absl::Status FsSpoolWriter::BuildDirectoryStructureIfNeeded() {
   if (!IsDirectory(spool_dir_)) {
     if (!IsDirectory(base_dir_)) {
       if (absl::Status status = MkDir(base_dir_); !status.ok()) {
-        return status; // failed to create base directory
+        return status;  // failed to create base directory
       }
     }
 
     if (absl::Status status = MkDir(spool_dir_); !status.ok()) {
-      return status; // failed to create spool directory;
+      return status;  // failed to create spool directory;
     }
   }
   if (!IsDirectory(tmp_dir_)) {
     // No need to check the base directory too, since spool_dir_ exists.
     if (absl::Status status = MkDir(tmp_dir_); !status.ok()) {
-      return status; // failed to create tmp directory
+      return status;  // failed to create tmp directory
     }
   }
   return absl::OkStatus();
@@ -197,7 +199,7 @@ std::string FsSpoolWriter::UniqueFilename() {
 
 absl::Status FsSpoolWriter::WriteMessage(absl::string_view msg) {
   if (absl::Status status = BuildDirectoryStructureIfNeeded(); !status.ok()) {
-    return status; // << "can't create directory structure for writer";
+    return status;  // << "can't create directory structure for writer";
   }
   // Flush messages to a file in the temporary directory.
   const std::string fname = UniqueFilename();
@@ -209,22 +211,23 @@ absl::Status FsSpoolWriter::WriteMessage(absl::string_view msg) {
   if (spool_size_estimate_ > max_spool_size_) {
     absl::StatusOr<size_t> estimate = EstimateDirSize(spool_dir_);
     if (!estimate.ok()) {
-      return estimate.status(); // failed to recompute spool size
+      return estimate.status();  // failed to recompute spool size
     }
     spool_size_estimate_ = *estimate;
     if (spool_size_estimate_ > max_spool_size_) {
       // Still over the limit: avoid writing.
-      return absl::UnavailableError("Spool size estimate greater than max allowed");
+      return absl::UnavailableError(
+          "Spool size estimate greater than max allowed");
     }
   }
   spool_size_estimate_ += EstimateDiskOccupation(msg.size());
 
   if (absl::Status status = WriteTmpFile(tmp_file, msg); !status.ok()) {
-    return status; // writing to temporary file
+    return status;  // writing to temporary file
   }
 
   if (absl::Status status = RenameFile(tmp_file, spool_file); !status.ok()) {
-    return status; // "moving tmp_file to the spooling area
+    return status;  // "moving tmp_file to the spooling area
   }
 
   return absl::OkStatus();

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.cc

@@ -14,10 +14,10 @@
 
 #include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h"
 
-#include <string>
-
 #include <os/log.h>
 
+#include <string>
+
 #include "absl/status/status.h"
 
 namespace fsspool {
@@ -31,7 +31,8 @@ FsSpoolLogBatchWriter::FsSpoolLogBatchWriter(FsSpoolWriter* fs_spool_writer,
 FsSpoolLogBatchWriter::~FsSpoolLogBatchWriter() {
   absl::Status s = FlushNoLock();
   if (!s.ok()) {
-    os_log(OS_LOG_DEFAULT, "Flush() failed with %s", s.ToString(absl::StatusToStringMode::kWithEverything).c_str());
+    os_log(OS_LOG_DEFAULT, "Flush() failed with %s",
+           s.ToString(absl::StatusToStringMode::kWithEverything).c_str());
     // LOG(WARNING) << "Flush() failed with " << s;
   }
 }

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h

@@ -60,7 +60,7 @@ class FsSpoolLogBatchWriter {
   FsSpoolWriter* writer_ ABSL_PT_GUARDED_BY(writer_mutex_);
   size_t max_batch_size_;
   absl::Mutex cache_mutex_;
-  binaryproto::LogBatch cache_ ABSL_GUARDED_BY(cache_mutex_);
+  santa::fsspool::binaryproto::LogBatch cache_ ABSL_GUARDED_BY(cache_mutex_);
 
   absl::Status FlushNoLock() ABSL_SHARED_LOCKS_REQUIRED(cache_mutex_);
 };

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_nowindows.cc

@@ -20,10 +20,10 @@
 #include <functional>
 #include <string>
 
-#include "absl/strings/match.h"
-#include "absl/strings/str_format.h"
 #include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h"
 #include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h"
+#include "absl/strings/match.h"
+#include "absl/strings/str_format.h"
 
 namespace fsspool {
 

Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h

@@ -18,6 +18,7 @@
 #include <functional>
 #include <string>
 
+#include "absl/status/status.h"
 #include "absl/strings/string_view.h"
 
 namespace fsspool {

Source/santad/testdata/protobuf/v1/allowlist.json

@@ -25,8 +25,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -53,6 +55,7 @@
  },
  "target": {
   "path": "close_file",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,

Source/santad/testdata/protobuf/v1/close.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -59,6 +61,7 @@
  },
  "target": {
   "path": "close_file",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,

Source/santad/testdata/protobuf/v1/exchangedata.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -59,6 +61,7 @@
  },
  "file1": {
   "path": "exchange_file_1",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,
@@ -86,6 +89,7 @@
  },
  "file2": {
   "path": "exchange_file_1",
+  "truncated": false,
   "stat": {
    "dev": 401,
    "mode": 402,

Source/santad/testdata/protobuf/v1/exec.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -95,6 +97,7 @@
   "cs_flags": 536871680,
   "executable": {
    "path": "fooexec",
+   "truncated": false,
    "stat": {
     "dev": 301,
     "mode": 302,

Source/santad/testdata/protobuf/v1/exit.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,

Source/santad/testdata/protobuf/v1/fork.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -87,8 +89,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo_child",
+   "truncated": false,
    "stat": {
     "dev": 301,
     "mode": 302,

Source/santad/testdata/protobuf/v1/link.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -59,6 +61,7 @@
  },
  "source": {
   "path": "source",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,

Source/santad/testdata/protobuf/v1/rename.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -59,6 +61,7 @@
  },
  "source": {
   "path": "source",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,
@@ -84,5 +87,6 @@
    "gen": 310
   }
  },
- "target": "target_dir/target_file"
+ "target": "target_dir/target_file",
+ "target_existed": false
 }

Source/santad/testdata/protobuf/v1/unlink.json

@@ -29,8 +29,10 @@
   },
   "is_platform_binary": true,
   "is_es_client": true,
+  "cs_flags": 0,
   "executable": {
    "path": "foo",
+   "truncated": false,
    "stat": {
     "dev": 101,
     "mode": 102,
@@ -59,6 +61,7 @@
  },
  "target": {
   "path": "unlink_file",
+  "truncated": false,
   "stat": {
    "dev": 301,
    "mode": 302,