Bugfix in timed writer. (Velocidex#1162)

Do not trucate the file for each write.

README.md

@@ -1,11 +1,11 @@
 # Velociraptor - Endpoint visibility and collection tool.
 
 Velociraptor is a tool for collecting host based state information
-using Velocidex Query Language (VQL) queries.
+using The Velociraptor Query Language (VQL) queries.
 
 To learn more about Velociraptor, read the documentation on:
 
-   https://www.velocidex.com/docs/
+https://docs.velociraptor.app/
 
 ## Quick start
 
@@ -24,13 +24,12 @@ collect artifacts from the client (which is just running on your own
 machine) as normal.
 
 Once you are ready for a full deployment, check out the various deployment options at
-https://www.velocidex.com/docs/getting-started
+https://docs.velociraptor.app/docs/overview/deployment/
 
 ## Training
 
-We have our complete training course (7 sessions x 2 hours each) on
-YouTube here
-https://www.youtube.com/playlist?list=PLz4xB83Y3Vbjtqr_ttOkBWZZ_ewEVVPXQ
+We have our complete training course (7 sessions x 2 hours each)
+https://docs.velociraptor.app/training/
 
 The course covers many aspects of Velociraptor in detail.
 
@@ -113,24 +112,35 @@ architectures before send us the PR.
 ## Supported platforms
 
 Velociraptor is written in Golang and so is available for all the
-platforms [supported by Go](https://github.com/golang/go/wiki/MinimumRequirements). This means that Windows XP and Windows server 2003 are **not** supported but anything after Windows 7/Vista is.
+platforms [supported by Go](https://github.com/golang/go/wiki/MinimumRequirements).
+This means that Windows XP and Windows server 2003 are **not**
+supported but anything after Windows 7/Vista is.
 
 We build our releases on Centos 6 (x64) for Linux and Sierra for MacOS
 so earlier platforms may not be supported by our release pipeline. If
 you need 32 bit builds you will need to build from source. You can do
 this easily by forking the project on GitHub, enabling GitHub Actions
 in your fork and editing the `Linux Build All Arches` pipeline.
 
+## Artifact Exchange
+
+Velociraptor's power comes from `VQL Artifacts`, that define many
+capabilities to collect many types of data from endpoints.
+Velociraptor comes with many built in `Artifacts` for the most common
+use cases. The community also maintains a large number of additional
+artifacts through the [Artifact Exchange](https://docs.velociraptor.app/exchange/).
+
 ## Getting help
 
 Questions and feedback are welcome at velociraptor-discuss@googlegroups.com
 
-You can also chat with us directly on discord https://www.velocidex.com/discord
+You can also chat with us directly on discord https://docs.velociraptor.app/discord
 
 File issues on https://github.com/Velocidex/velociraptor
 
 Read more about Velociraptor on our blog:
-
-https://www.velocidex.com/blog/
+https://docs.velociraptor.app/blog/
 
 Hang out on Medium https://medium.com/velociraptor-ir
+
+Follow us on Twitter @velocidex

api/csv.go

@@ -189,10 +189,7 @@ func getEventTableWithPathManager(
 	}
 
 	if in.EndTime != 0 {
-		err = rs_reader.SeekToTime(time.Unix(int64(in.EndTime), 0))
-		if err != nil {
-			return nil, err
-		}
+		rs_reader.SetMaxTime(time.Unix(int64(in.EndTime), 0))
 	}
 
 	// Unpack the rows into the output protobuf

result_sets/timed/reader.go

@@ -3,6 +3,7 @@ package timed
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"time"
 
@@ -68,7 +69,16 @@ func (self *TimedResultSetReader) GetAvailableFiles(
 	return self.files
 }
 
+func (self *TimedResultSetReader) Debug() {
+	fmt.Printf("Current idx %v\n", self.current_files_idx)
+	for _, file := range self.files {
+		fmt.Printf("%v %v-%v\n", file.Path, file.StartTime, file.EndTime)
+	}
+}
+
 func (self *TimedResultSetReader) SeekToTime(offset time.Time) error {
+	self.Close()
+
 	self.start = offset
 	for idx, file := range self.files {
 		if offset.Before(file.StartTime) {
@@ -77,7 +87,8 @@ func (self *TimedResultSetReader) SeekToTime(offset time.Time) error {
 		}
 
 		// This file spans the required time
-		if offset.After(file.StartTime) && offset.Before(file.EndTime) {
+		if (offset.Equal(file.StartTime) || offset.After(file.StartTime)) &&
+			offset.Before(file.EndTime) {
 			self.current_files_idx = idx
 
 			reader, err := self.getReader()
@@ -106,7 +117,12 @@ func (self *TimedResultSetReader) SetMaxTime(end time.Time) {
 	self.end = end
 }
 
-func (self *TimedResultSetReader) Close() {}
+func (self *TimedResultSetReader) Close() {
+	if self.current_reader != nil {
+		self.current_reader.Close()
+		self.current_reader = nil
+	}
+}
 
 func (self *TimedResultSetReader) getReader() (*timelines.TimelineReader, error) {
 	if self.current_reader != nil {
@@ -133,7 +149,6 @@ func (self *TimedResultSetReader) getReader() (*timelines.TimelineReader, error)
 			}
 		}
 
-		self.current_files_idx++
 		self.current_reader = reader
 		return reader, nil
 	}
@@ -159,7 +174,8 @@ func (self *TimedResultSetReader) maybeUpgradeIndex(
 	new_path := path_manager.Path() + ".tmp"
 	tmp_path_manager := timelinePathManager(new_path)
 	tmp_writer, err := timelines.NewTimelineWriter(
-		self.file_store_factory, tmp_path_manager)
+		self.file_store_factory, tmp_path_manager,
+		true /* truncate */)
 	if err != nil {
 		return nil, err
 	}
@@ -211,8 +227,8 @@ func (self *TimedResultSetReader) Rows(
 
 			// When the reader is exhausted reset it so
 			// next getReader() can pick the next reader.
-			self.current_reader.Close()
-			self.current_reader = nil
+			self.Close()
+			self.current_files_idx++
 		}
 	}()
 

result_sets/timed/writer.go

@@ -90,8 +90,9 @@ func (self *TimedResultSetWriterImpl) getWriter(ts time.Time) (
 		return self.writer, nil
 	}
 
-	writer, err := timelines.NewTimelineWriter(self.file_store_factory,
-		timelinePathManager(log_path))
+	writer, err := timelines.NewTimelineWriter(
+		self.file_store_factory,
+		timelinePathManager(log_path), false /* truncate */)
 	if err != nil {
 		return nil, err
 	}

timelines/supertimeline.go

@@ -163,7 +163,7 @@ func (self *SuperTimelineWriter) AddChild(name string) (*TimelineWriter, error)
 	new_timeline_path_manager := self.path_manager.GetChild(name)
 	file_store_factory := file_store.GetFileStore(self.config_obj)
 	writer, err := NewTimelineWriter(
-		file_store_factory, new_timeline_path_manager)
+		file_store_factory, new_timeline_path_manager, true /* truncate */)
 	if err != nil {
 		return nil, err
 	}

timelines/timelines_test.go

@@ -89,7 +89,7 @@ func (self *TimelineTestSuite) TestSuperTimelineWriter() {
 func (self *TimelineTestSuite) TestTimelineWriter() {
 	path_manager := &TimelinePathManager{"T.1234", "Test"}
 	file_store_factory := file_store.GetFileStore(self.config_obj)
-	timeline, err := NewTimelineWriter(file_store_factory, path_manager)
+	timeline, err := NewTimelineWriter(file_store_factory, path_manager, true /* truncate */)
 	assert.NoError(self.T(), err)
 
 	for i := int64(0); i <= 10; i++ {

timelines/writer.go

@@ -82,21 +82,25 @@ func (self *TimelineWriter) Close() {
 
 func NewTimelineWriter(
 	file_store_factory api.FileStore,
-	path_manager TimelinePathManagerInterface) (*TimelineWriter, error) {
+	path_manager TimelinePathManagerInterface,
+	truncate bool) (*TimelineWriter, error) {
 	fd, err := file_store_factory.WriteFile(
 		path_manager.Path())
 	if err != nil {
 		return nil, err
 	}
-	fd.Truncate()
 
 	index_fd, err := file_store_factory.WriteFile(
 		path_manager.Index())
 	if err != nil {
 		fd.Close()
 		return nil, err
 	}
-	index_fd.Truncate()
+
+	if truncate {
+		fd.Truncate()
+		index_fd.Truncate()
+	}
 
 	return &TimelineWriter{fd: fd, index_fd: index_fd}, nil