Uploaded files are now shows with client paths

actions/proto/transport.proto

@@ -26,6 +26,8 @@ message PathSpec {
             "This value is interpreted by the accessor in its own way.",
         }];
 
+    repeated string components = 4;
+
     string accessor = 3 [(sem_type) = {
             description: "The accessor used to retrieve the file.",
         }];

api/download.go

@@ -76,13 +76,24 @@ func returnError(w http.ResponseWriter, code int, message string) {
 }
 
 type vfsFileDownloadRequest struct {
-	ClientId   string   `schema:"client_id"`
-	VfsPath    string   `schema:"vfs_path"`
+	ClientId string `schema:"client_id"`
+
+	// This is the path within the client VFS in the usual client path
+	// notation - this is what is seen in the uploads table. We use
+	// this field to determine the download attachment name.
+	VfsPath string `schema:"vfs_path"`
+
+	// in the VFS abstration these components are given as the path
+	// within the VFS UI. We use these to open the VFSDownloadInfoPath
+	// datastore item to figure out where the actual file is stored.
 	Components []string `schema:"components[]"`
-	Offset     int64    `schema:"offset"`
-	Length     int      `schema:"length"`
-	Encoding   string   `schema:"encoding"`
-	OrgId      string   `schema:"org_id"`
+
+	// This is the file store path to fetch.
+	FSComponents []string `schema:"fs_components"`
+	Offset       int64    `schema:"offset"`
+	Length       int      `schema:"length"`
+	Encoding     string   `schema:"encoding"`
+	OrgId        string   `schema:"org_id"`
 }
 
 // URL format: /api/v1/DownloadVFSFile
@@ -116,17 +127,30 @@ func vfsFileDownloadHandler() http.Handler {
 		}
 
 		var path_spec api.FSPathSpec
+		filename := "upload.bin"
+
 		client_path_manager := paths.NewClientPathManager(request.ClientId)
 
-		// Uploads table has direct vfs paths
-		if request.VfsPath != "" {
+		// Newer API calls pass the filestore components directly
+		if len(request.FSComponents) > 0 {
+			path_spec = path_specs.NewUnsafeFilestorePath(request.FSComponents...).
+				SetType(api.PATH_TYPE_FILESTORE_ANY)
+
+			base := utils.Base(request.VfsPath)
+			filename = strings.Replace(base, "\"", "_", -1)
+
+			// Uploads table has direct vfs paths
+		} else if request.VfsPath != "" {
 			path_spec, err = client_path_manager.GetUploadsFileFromVFSPath(
 				request.VfsPath)
 			if err != nil {
 				returnError(w, 404, err.Error())
 				return
 			}
+			filename = strings.Replace(path_spec.Base(), "\"", "_", -1)
 
+			// Use the Components field to fetch the
+			// VFSDownloadInfoPath record for this directory.
 		} else {
 			db, err := datastore.GetDB(org_config_obj)
 			if err != nil {
@@ -147,6 +171,7 @@ func vfsFileDownloadHandler() http.Handler {
 				download_info.Components...).
 				SetType(api.PATH_TYPE_FILESTORE_ANY)
 
+			filename = strings.Replace(path_spec.Base(), "\"", "_", -1)
 		}
 
 		file, err := file_store.GetFileStore(org_config_obj).ReadFile(path_spec)
@@ -175,40 +200,51 @@ func vfsFileDownloadHandler() http.Handler {
 
 		offset := request.Offset
 
-		// From here on we sent the headers and we can not
-		// really report an error to the client.
-		filename := strings.Replace(path_spec.Base(), "\"", "_", -1)
-		w.Header().Set("Content-Disposition", "attachment; filename="+
-			url.PathEscape(filename))
-		w.Header().Set("Content-Type", "binary/octet-stream")
-		w.WriteHeader(200)
-
+		// Read the first buffer now so we can report errors
 		length_sent := 0
+		headers_sent := false
+
 		buf := pool.Get().([]byte)
 		defer pool.Put(buf)
 
 		for {
-			n, _ := reader_at.ReadAt(buf, offset)
-			if n > 0 {
-				if request.Length != 0 {
-					length_to_send := request.Length - length_sent
-					if n > length_to_send {
-						n = length_to_send
-					}
+			n, err := reader_at.ReadAt(buf, offset)
+			if err != nil && err != io.EOF {
+				// Only send errors if the headers have not yet been
+				// sent.
+				if !headers_sent {
+					returnError(w, 500, err.Error())
+					headers_sent = true
 				}
-				if n == 0 {
-					return
+				return
+			}
+			if request.Length != 0 {
+				length_to_send := request.Length - length_sent
+				if n > length_to_send {
+					n = length_to_send
 				}
+			}
+			if n <= 0 {
+				return
+			}
 
-				_, err := w.Write(buf[:n])
-				if err != nil {
-					return
-				}
-				length_sent += n
-				offset += int64(n)
-			} else {
+			// Write an ok status which includes the attachment name
+			// but only if no other data was sent.
+			if !headers_sent {
+				w.Header().Set("Content-Disposition", "attachment; filename="+
+					url.PathEscape(filename))
+				w.Header().Set("Content-Type", "binary/octet-stream")
+				w.WriteHeader(200)
+				headers_sent = true
+			}
+
+			written, err := w.Write(buf[:n])
+			if err != nil {
 				return
 			}
+
+			length_sent += written
+			offset += int64(n)
 		}
 	})
 }

artifacts/testdata/server/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads.json

@@ -1 +1,2 @@
-{"Timestamp":1573198259,"started":"2019-11-08 07:30:59.920512962 +0000 UTC","vfs_path":"/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/1.zip","expected_size":1319}
+{"Timestamp":1573198259,"started":"2019-11-08 07:30:59.920512962 +0000 UTC","vfs_path":"C:\\1.zip","_Components":["clients","C.4f5e52adf0a337a9","collections","F.BN2HJCPOF5U7U","uploads","auto","C:","1.zip"],"file_size":1319,"uploaded_size":1319}
+{"Timestamp":1573198259,"started":"2019-11-08 07:30:59.920512962 +0000 UTC","vfs_path":"/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip","expected_size":1319}

artifacts/testdata/server/testcases/artifacts.out.yaml

@@ -2,9 +2,28 @@ SELECT *, basename(path=file_store(path=vfs_path)) FROM uploads(client_id='C.4f5
  {
   "Timestamp": 1573198259,
   "started": "2019-11-08 07:30:59.920512962 +0000 UTC",
-  "vfs_path": "fs:/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/1.zip",
-  "expected_size": 1319,
+  "vfs_path": "fs:/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/auto/C:/1.zip",
+  "_Components": [
+   "clients",
+   "C.4f5e52adf0a337a9",
+   "collections",
+   "F.BN2HJCPOF5U7U",
+   "uploads",
+   "auto",
+   "C:",
+   "1.zip"
+  ],
+  "file_size": 1319,
+  "uploaded_size": 1319,
+  "client_path": "C:\\1.zip",
   "basename(path=file_store(path=vfs_path))": "1.zip"
+ },
+ {
+  "Timestamp": 1573198259,
+  "started": "2019-11-08 07:30:59.920512962 +0000 UTC",
+  "vfs_path": "fs:/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip",
+  "expected_size": 1319,
+  "basename(path=file_store(path=vfs_path))": "old_style.zip"
  }
 ]SELECT collect_client( client_id='C.11a3013ccaXXXXX', artifacts='Windows.KapeFiles.Targets', env=dict(Device ='C:', VSSAnalysis='Y', KapeTriage='Y')).request AS Flow FROM scope()[
  {

flows/artifacts.go

@@ -292,6 +292,7 @@ func flushContextUploadedFiles(
 			Set("Timestamp", time.Now().UTC().Unix()).
 			Set("started", time.Now().UTC().String()).
 			Set("vfs_path", row.Name).
+			Set("_Components", row.Components).
 			Set("file_size", row.Size).
 			Set("uploaded_size", row.StoredSize))
 	}
@@ -529,7 +530,7 @@ func appendUploadDataToFile(
 		collection_context.UploadedFiles = append(
 			collection_context.UploadedFiles,
 			&flows_proto.ArtifactUploadedFileInfo{
-				Name:       file_path_manager.Path().AsClientPath(),
+				Name:       file_path_manager.VisibleVFSPath(),
 				Components: file_path_manager.Path().Components(),
 				Size:       file_buffer.Size,
 				StoredSize: size,
@@ -573,10 +574,8 @@ func appendUploadDataToFile(
 		collection_context.UploadedFiles = append(
 			collection_context.UploadedFiles,
 			&flows_proto.ArtifactUploadedFileInfo{
-				Name: file_path_manager.IndexPath().
-					AsClientPath(),
-				Components: file_path_manager.IndexPath().
-					Components(),
+				Name:       file_path_manager.VisibleVFSPath() + ".idx",
+				Components: file_path_manager.IndexPath().Components(),
 				Size:       uint64(len(data)),
 				StoredSize: uint64(len(data)),
 			})