diff --git a/artifacts/definitions/Server/Utils/CreateCollector.yaml b/artifacts/definitions/Server/Utils/CreateCollector.yaml index 07690be37f0..9c974873007 100644 --- a/artifacts/definitions/Server/Utils/CreateCollector.yaml +++ b/artifacts/definitions/Server/Utils/CreateCollector.yaml @@ -48,10 +48,10 @@ parameters: - name: template default: description: The HTML report template to use. - + - name: encryption_scheme description: | - Encryption scheme to use. Currently supported are Passowrd and PGP + Encryption scheme to use. Currently supported are Passowrd, X509 or PGP - name: encryption_args description: | @@ -121,7 +121,9 @@ parameters: - name: opt_cpu_limit default: "0" type: int - description: A number between 0 to 100 representing the target maximum CPU utilization during running of this artifact. + description: | + A number between 0 to 100 representing the target maximum CPU + utilization during running of this artifact. - name: opt_progress_timeout default: "1800" @@ -141,28 +143,9 @@ parameters: - name: StandardCollection type: hidden default: | - // Add all the tools we are going to use to the inventory. - LET _ <= SELECT inventory_add(tool=ToolName, hash=ExpectedHash) - FROM parse_csv(filename="/inventory.csv", accessor="me") - WHERE log(message="Adding tool " + ToolName) - - LET baseline <= SELECT Fqdn FROM info() - - // Make the filename safe on windows but we trust the OutputPrefix. - LET filename <= OutputPrefix + regex_replace( - source=format(format="Collection-%s-%s", - args=[baseline[0].Fqdn, - timestamp(epoch=now()).MarshalText]), - re="[^0-9A-Za-z\\-]", replace="_") - LET _ <= log(message="Will collect package " + filename) LET report_filename <= if(condition=Template, then=filename + ".html") - LET X <= SELECT format(format="%02x", args=rand(range=255)) AS A FROM range(end=25) - LET pass = SELECT * FROM switch(a={SELECT join(array=X.A) as Pass From scope() WHERE encryption_scheme =~ "pgp|x509"}, - b={SELECT encryption_args.password as Pass FROM scope() WHERE encryption_scheme =~ "password"}, - c={SELECT Null as Pass FROM scope()}) - SELECT * FROM collect(artifacts=Artifacts, report=report_filename, args=Parameters, output=filename + ".zip", template=Template, cpu_limit=CpuLimit, @@ -171,13 +154,7 @@ parameters: password=pass[0].Pass, level=Level, format=Format, - metadata=if(condition=encryption_args.public_key, then={ - SELECT pk_encrypt(data=pass[0].Pass, public_key=encryption_args.public_key, - scheme=encryption_scheme) AS EncryptedPass, - encryption_scheme as Scheme, - encryption_args.public_key as PublicKey FROM scope() - }) - ) + metadata=ContainerMetadata) - name: S3Collection type: hidden @@ -220,16 +197,15 @@ parameters: endpoint=TargetArgs.endpoint, hostkey = TargetArgs.hostkey) - - name: CloudCollection + - name: CommonCollections type: hidden default: | // Add all the tools we are going to use to the inventory. LET _ <= SELECT inventory_add(tool=ToolName, hash=ExpectedHash) - FROM parse_csv(filename="/inventory.csv", accessor="me") - WHERE log(message="Adding tool " + ToolName) + FROM parse_csv(filename="/inventory.csv", accessor="me") + WHERE log(message="Adding tool " + ToolName) LET baseline <= SELECT Fqdn, basename(path=Exe) AS Exe FROM info() - LET TargetArgs <= target_args // Make the filename safe on windows but we trust the OutputPrefix. LET filename <= OutputPrefix + regex_replace( @@ -238,6 +214,47 @@ parameters: timestamp(epoch=now()).MarshalText]), re="[^0-9A-Za-z\\-]", replace="_") + -- Make a random hex string as a random password + LET RandomPassword <= SELECT format(format="%02x", + args=rand(range=255)) AS A + FROM range(end=25) + + LET pass = SELECT * FROM switch(a={ + + -- For X509 encryption we use a random session password. + SELECT join(array=RandomPassword.A) as Pass From scope() + WHERE encryption_scheme =~ "pgp|x509" + AND log(message="I will generate a container password using the %v scheme", + args=encryption_scheme) + + }, b={ + + -- Otherwise the user specified the password. + SELECT encryption_args.password as Pass FROM scope() + WHERE encryption_scheme =~ "password" + + }, c={ + + -- No password specified. + SELECT Null as Pass FROM scope() + }) + + -- For X509 encryption_scheme, store the encrypted + -- password in the metadata file for later retrieval. + LET ContainerMetadata = if( + condition=encryption_args.public_key, + then=dict( + EncryptedPass=pk_encrypt(data=pass[0].Pass, + public_key=encryption_args.public_key, + scheme=encryption_scheme), + Scheme=encryption_scheme, + PublicKey=encryption_args.public_key)) + + - name: CloudCollection + type: hidden + default: | + LET TargetArgs <= target_args + // Try to upload the log file now to see if we are even able to // upload at all - we do this to avoid having to collect all the // data and then failing the upload step. @@ -251,11 +268,6 @@ parameters: " and upload to cloud bucket " + TargetArgs.bucket) LET report_filename <= if(condition=Template, then=tempfile(extension=".html")) - LET X <= SELECT format(format="%02x", args=rand(range=255)) AS A FROM range(end=25) - LET pass = SELECT * FROM switch(a={SELECT join(array=X.A) as Pass From scope() WHERE encryption_scheme =~ "pgp"}, - b={SELECT encryption_args.password as Pass FROM scope() WHERE encryption_scheme =~ "password"}, - c={SELECT Null as Pass FROM scope()}) - LET collect_and_upload = SELECT upload_file(filename=Container, name=filename+".zip", @@ -267,6 +279,7 @@ parameters: upload_file(filename=baseline[0].Exe + ".log", name=filename+".log", accessor="file") AS LogUpload + FROM collect(artifacts=Artifacts, report=report_filename, args=Parameters, @@ -278,15 +291,13 @@ parameters: timeout=Timeout, password=pass[0].Pass, level=Level, - metadata=if(condition=encryption_args.public_key, then={ - SELECT pk_encrypt(data=pass.Pass, public_key=encryption_args.public_key, scheme=encryption_scheme) AS EncryptedPass, encryption_scheme as Scheme FROM scope() - }) - ) + metadata=ContainerMetadata) SELECT * FROM if(condition=upload_test.Path, then=collect_and_upload, - else={SELECT log(message="Aborting collection: Failed to upload to cloud bucket!") - FROM scope()}) + else={SELECT log( + message="Aborting collection: Failed to upload to cloud bucket!") + FROM scope()}) - name: PackageToolsArtifact description: Collects and uploads third party binaries. @@ -392,20 +403,35 @@ sources: artifact_definitions=PackageToolsArtifact) LET CollectionArtifact <= SELECT Value FROM switch( - a = { SELECT StandardCollection AS Value FROM scope() WHERE target = "ZIP" }, - b = { SELECT S3Collection + CloudCollection AS Value FROM scope() WHERE target = "S3" }, - c = { SELECT GCSCollection + CloudCollection AS Value FROM scope() WHERE target = "GCS" }, - d = { SELECT SFTPCollection + CloudCollection AS Value FROM scope() WHERE target = "SFTP" }, - e = { SELECT "" AS Value FROM scope() WHERE log(message="Unknown collection type " + target) } + a = { SELECT CommonCollections + StandardCollection AS Value + FROM scope() + WHERE target = "ZIP" }, + b = { SELECT S3Collection + CommonCollections + CloudCollection AS Value + FROM scope() + WHERE target = "S3" }, + c = { SELECT GCSCollection + CommonCollections + CloudCollection AS Value + FROM scope() + WHERE target = "GCS" }, + d = { SELECT SFTPCollection + CommonCollections + CloudCollection AS Value + FROM scope() + WHERE target = "SFTP" }, + e = { SELECT "" AS Value FROM scope() + WHERE log(message="Unknown collection type " + target) } ) - - LET use_server_cert = SELECT log(message="Pubkey encryption specified, but no cert/key provided. Defaulting to server frontend cert") FROM scope() WHERE encryption_scheme =~ "x509" AND encryption_args.public_key =~ "" - LET updated_encryption_args <= if(condition=use_server_cert, - then=dict(public_key=server_frontend_cert(), - scheme="x509"), - else=encryption_args - ) + LET use_server_cert = encryption_scheme =~ "x509" + AND encryption_args.public_key =~ "" + AND log(message="Pubkey encryption specified, but no cert/key provided. Defaulting to server frontend cert") + + -- For x509, if no public key cert is specified, we use the + -- server's own key. This makes it easy for the server to import + -- the file again. + LET updated_encryption_args <= if( + condition=use_server_cert, + then=dict(public_key=server_frontend_cert(), + scheme="x509"), + else=encryption_args + ) LET definitions <= SELECT * FROM chain( a = { SELECT name, description, tools, parameters, sources, reports @@ -433,7 +459,7 @@ sources: type="json"), dict(name="Template", default=template), dict(name="encryption_scheme", default=encryption_scheme), - dict(name="encryption_args", + dict(name="encryption_args", default=serialize(format='json', item=updated_encryption_args), type="json" ), diff --git a/bin/collector_test.go b/bin/collector_test.go index 21c5fc3b9c0..b926737cd00 100644 --- a/bin/collector_test.go +++ b/bin/collector_test.go @@ -18,6 +18,7 @@ import ( "github.com/Velocidex/yaml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" @@ -35,16 +36,26 @@ func init() { } type CollectorTestSuite struct { + suite.Suite + binary string extension string tmpdir string config_file string config_obj *config_proto.Config test_server *httptest.Server + + OS_TYPE string } -func CollectorSetupTest(t *testing.T) *CollectorTestSuite { - self := &CollectorTestSuite{} +func (self *CollectorTestSuite) SetupSuite() { + self.findAndPrepareBinary() + self.addArtifactDefinitions() + self.uploadToolDefinitions() +} + +func (self *CollectorTestSuite) findAndPrepareBinary() { + t := self.T() if runtime.GOOS == "windows" { self.extension = ".exe" @@ -93,7 +104,7 @@ func CollectorSetupTest(t *testing.T) *CollectorTestSuite { // Start a web server that serves the filesystem - NOTE: Normally // this would be served from the Velociraptor server itself but // here we dont want to start it so we serve simple HTTP server - // and require all tools to be served remotes from these URL. + // and require all tools to be served remotely from these URL. self.test_server = httptest.NewServer( http.FileServer(http.Dir(self.tmpdir))) @@ -108,34 +119,20 @@ func CollectorSetupTest(t *testing.T) *CollectorTestSuite { fd.Write(serialized) fd.Close() - return self -} - -func (self *CollectorTestSuite) TearDownTest() { - os.RemoveAll(self.tmpdir) - self.test_server.Close() -} - -func TestCollector(t *testing.T) { - t.Parallel() - - self := CollectorSetupTest(t) - defer self.TearDownTest() - - OS_TYPE := "Linux" + // Record the OS Type we are running on. + self.OS_TYPE = "Linux" if runtime.GOOS == "windows" { - OS_TYPE = "Windows" + self.OS_TYPE = "Windows" } else if runtime.GOOS == "darwin" { - OS_TYPE = "Darwin" + self.OS_TYPE = "Darwin" } +} - // Change into the tmpdir - old_dir, _ := os.Getwd() - defer os.Chdir(old_dir) - - os.Chdir(self.tmpdir) +func (self *CollectorTestSuite) addArtifactDefinitions() { + t := self.T() - // Create a new artifact.. + // Create new artifacts and just save them on the filesystem - we + // dont need a real repository manager. file_store_factory := file_store.GetFileStore(self.config_obj) fd, err := file_store_factory.WriteFile(paths.GetArtifactDefintionPath( @@ -210,37 +207,40 @@ reports: {{ end }} `)) fd.Close() - cmd := exec.Command(self.binary, "--config", self.config_file, - "artifacts", "show", "Custom.TestArtifact") - out, err := cmd.CombinedOutput() - fmt.Println(string(out)) - require.NoError(t, err) - var os_name string - for _, os_name = range []string{"Windows", "Windows_x86", "Linux", "Darwin"} { - cmd = exec.Command(self.binary, "--config", self.config_file, + fd, err = file_store_factory.WriteFile( + paths.GetArtifactDefintionPath("Custom.TestHello")) + assert.NoError(t, err) + + fd.Truncate() + fd.Write([]byte(`name: Custom.TestHello +sources: + - query: SELECT "Hello" AS Hi FROM scope() +`)) + fd.Close() +} + +func (self *CollectorTestSuite) uploadToolDefinitions() { + t := self.T() + + // Upload a small file (the config file) for all the other + // architectures. + for _, os_name := range []string{"Windows", "Windows_x86", "Linux", "Darwin"} { + cmd := exec.Command(self.binary, "--config", self.config_file, "tools", "upload", "--name", "Velociraptor"+os_name, self.config_file, "--serve_remote") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() fmt.Println(string(out)) require.NoError(t, err) } - switch runtime.GOOS { - case "windows": - os_name = "Windows" - case "linux": - os_name = "Linux" - case "darwin": - os_name = "Darwin" - } - - cmd = exec.Command(self.binary, "--config", self.config_file, - "tools", "upload", "--name", "Velociraptor"+os_name, + // Upload the real thing for the architecture we are running on. + cmd := exec.Command(self.binary, "--config", self.config_file, + "tools", "upload", "--name", "Velociraptor"+self.OS_TYPE, self.test_server.URL+"/"+filepath.Base(self.binary), "--serve_remote") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() fmt.Println(string(out)) require.NoError(t, err) @@ -252,6 +252,8 @@ reports: assert.NotRegexp(t, "serve_locally", string(out)) assert.NotRegexp(t, "hash: .+", string(out)) + // Add ourselves again as a tool called MyTool - the artifact will + // call it. cmd = exec.Command(self.binary, "--config", self.config_file, "tools", "upload", "--name", "MyTool", self.test_server.URL+"/"+filepath.Base(self.binary), @@ -259,6 +261,21 @@ reports: out, err = cmd.CombinedOutput() fmt.Println(string(out)) require.NoError(t, err) +} + +func (self *CollectorTestSuite) TearDownSuite() { + os.RemoveAll(self.tmpdir) + self.test_server.Close() +} + +func (self *CollectorTestSuite) TestCollector() { + t := self.T() + + // Change into the tmpdir + old_dir, _ := os.Getwd() + defer os.Chdir(old_dir) + + os.Chdir(self.tmpdir) // Create an embedded data file data_file_name := filepath.Join(self.tmpdir, "test.yar") @@ -270,21 +287,23 @@ reports: fd.Close() } - cmd = exec.Command(self.binary, "--config", self.config_file, + // Add it as a tool + cmd := exec.Command(self.binary, "--config", self.config_file, "tools", "upload", "--name", "MyDataFile", self.test_server.URL+"/test.yar", "--serve_remote") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() fmt.Println(string(out)) require.NoError(t, err) + // Where we will write the collection. output_zip := filepath.Join(self.tmpdir, "output.zip") // Now we want to create a stand alone collector. We do this // by collecting the Server.Utils.CreateCollector artifact cmdline := []string{"--config", self.config_file, "-v", "artifacts", "collect", "Server.Utils.CreateCollector", - "--args", "OS=" + OS_TYPE, + "--args", "OS=" + self.OS_TYPE, "--args", "artifacts=[\"Custom.TestArtifact\"]", "--args", "parameters={\"Custom.TestArtifact\":{\"MyParameter\": \"MyValue\"}}", "--args", "target=ZIP", @@ -299,6 +318,7 @@ reports: fmt.Println(string(out)) require.NoError(t, err) + // Inspect the resulting binary - it should have a zip appended. r, err := zip.OpenReader(output_zip) assert.NoError(t, err) @@ -329,14 +349,14 @@ reports: } } - // Now just run the executable. + // Now just run the executable to check the config. fmt.Printf("Config show\n") cmd = exec.Command(output_executable, "config", "show") out, err = cmd.CombinedOutput() fmt.Println(string(out)) require.NoError(t, err) - // Now just run the executable. + // Run the executable and see what it collects. cmd = exec.Command(output_executable) out, err = cmd.CombinedOutput() fmt.Println(string(out)) @@ -347,12 +367,19 @@ reports: assert.NoError(t, err) assert.Equal(t, 1, len(zip_files)) + // Clean it up after we are done. + defer func() { + err := os.Remove(zip_files[0]) + assert.NoError(t, err) + }() + // Inspect the collection zip file - there should be a single // artifact output from our custom artifact, and the data it // produces should have the string Foobar in it. r, err = zip.OpenReader(zip_files[0]) assert.NoError(t, err) + defer r.Close() assert.True(t, len(r.File) > 0) for _, f := range r.File { @@ -408,3 +435,120 @@ reports: // fmt.Println(string(data)) } + +// Check that we can properly generated encrypted containers. +func (self *CollectorTestSuite) TestCollectorEncrypted() { + t := self.T() + + // Change into the tmpdir + old_dir, _ := os.Getwd() + defer os.Chdir(old_dir) + + os.Chdir(self.tmpdir) + + output_zip := filepath.Join(self.tmpdir, "output_enc.zip") + + // Now we want to create a stand alone collector. We do this + // by collecting the Server.Utils.CreateCollector artifact + cmdline := []string{"--config", self.config_file, "-v", + "artifacts", "collect", "Server.Utils.CreateCollector", + "--args", "OS=" + self.OS_TYPE, + "--args", "artifacts=[\"Custom.TestHello\"]", + "--args", "parameters={\"Custom.TestHello\":{\"MyParameter\": \"MyValue\"}}", + "--args", "target=ZIP", + "--args", "opt_admin=N", + "--args", "opt_prompt=N", + "--args", "encryption_scheme=X509", + "--args", `encryption_args={"public_key":"","password":""}`, + "--output", output_zip, + } + + cmd := exec.Command(self.binary, cmdline...) + out, err := cmd.CombinedOutput() + fmt.Println(string(out)) + require.NoError(t, err) + + r, err := zip.OpenReader(output_zip) + assert.NoError(t, err) + + defer r.Close() + + output_executable := filepath.Join(self.tmpdir, "collector"+self.extension) + for _, f := range r.File { + fmt.Printf("Contents of collector: %s (%v bytes)\n", + f.Name, f.UncompressedSize) + if strings.HasPrefix(f.Name, "Collector") { + fmt.Printf("Extracting %v to %v\n", f.Name, output_executable) + + rc, err := f.Open() + assert.NoError(t, err) + + out_fd, err := os.OpenFile( + output_executable, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700) + assert.NoError(t, err) + + n, err := io.Copy(out_fd, rc) + assert.NoError(t, err) + rc.Close() + out_fd.Close() + + fmt.Printf("Copied %v bytes\n", n) + } + } + + // Now just run the executable. + cmd = exec.Command(output_executable) + out, err = cmd.CombinedOutput() + fmt.Println(string(out)) + require.NoError(t, err) + + // There should be a collection now. + zip_files, err := filepath.Glob("Collection-*.zip") + assert.NoError(t, err) + assert.Equal(t, 1, len(zip_files)) + + // Clean up after we are done. + defer func() { + err := os.Remove(zip_files[0]) + assert.NoError(t, err) + }() + + // Inspect the collection zip file - The zip file is encrypted + // therefore contains only a single metadata file (with the + // encrypted session key) and an opaque data.zip member which is + // password encrypted. + r, err = zip.OpenReader(zip_files[0]) + assert.NoError(t, err) + + defer r.Close() + + assert.True(t, len(r.File) > 0) + + names := []string{} + for _, f := range r.File { + fmt.Printf("Contents of %s:\n", f.Name) + names = append(names, f.Name) + + switch f.Name { + case "metadata.json": + rc, err := f.Open() + assert.NoError(t, err) + + data, err := ioutil.ReadAll(rc) + assert.NoError(t, err) + + // The metadata should contain information required to unpack + // the zip. + assert.Contains(t, string(data), "EncryptedPass") + + // Encryption scheme. + assert.Contains(t, string(data), `"Scheme": "X509"`) + } + } + + assert.Equal(t, []string{"metadata.json", "data.zip"}, names) +} + +func TestCollector(t *testing.T) { + suite.Run(t, &CollectorTestSuite{}) +} diff --git a/bin/deaddisk_test.go b/bin/deaddisk_test.go index 91924f1ffa1..d53726644c2 100644 --- a/bin/deaddisk_test.go +++ b/bin/deaddisk_test.go @@ -4,15 +4,13 @@ import ( "os" "os/exec" "path/filepath" - "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestDeaddisk(t *testing.T) { - self := CollectorSetupTest(t) - defer self.TearDownTest() +func (self *CollectorTestSuite) TestDeaddisk() { + t := self.T() // Create a "Windows" directory in the tmpdir windows_dir := filepath.Join(self.tmpdir, "Windows") diff --git a/gui/velociraptor/src/components/flows/new-collection.js b/gui/velociraptor/src/components/flows/new-collection.js index 4335c23ab2e..d96d03c995f 100644 --- a/gui/velociraptor/src/components/flows/new-collection.js +++ b/gui/velociraptor/src/components/flows/new-collection.js @@ -21,7 +21,6 @@ import Spinner from '../utils/spinner.js'; import Col from 'react-bootstrap/Col'; import StepWizard from 'react-step-wizard'; -import VeloForm from '../forms/form.js'; import ValidatedInteger from "../forms/validated_int.js"; diff --git a/gui/velociraptor/src/components/flows/new-collections-parameters.js b/gui/velociraptor/src/components/flows/new-collections-parameters.js index 6a43a31b09b..6c8188f2697 100644 --- a/gui/velociraptor/src/components/flows/new-collections-parameters.js +++ b/gui/velociraptor/src/components/flows/new-collections-parameters.js @@ -8,7 +8,6 @@ import Button from 'react-bootstrap/Button'; import ButtonGroup from 'react-bootstrap/ButtonGroup'; import Row from 'react-bootstrap/Row'; import Col from 'react-bootstrap/Col'; -import Form from 'react-bootstrap/Form'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; diff --git a/gui/velociraptor/src/components/flows/offline-collector.js b/gui/velociraptor/src/components/flows/offline-collector.js index aaec587eb56..dc2c14e32d1 100644 --- a/gui/velociraptor/src/components/flows/offline-collector.js +++ b/gui/velociraptor/src/components/flows/offline-collector.js @@ -104,22 +104,23 @@ class OfflineCollectorParameters extends React.Component { - {(this.props.parameters.encryption_scheme == "PGP" || this.props.parameters.encryption_scheme == "X509") && <> - - {T("Public Key/Cert")} - - { - this.props.parameters.encryption_args.public_key = e.target.value; - this.props.setParameters(this.props.parameters); - }} /> - - + {(this.props.parameters.encryption_scheme === "PGP" || + this.props.parameters.encryption_scheme === "X509") && + + {T("Public Key/Cert")} + + { + this.props.parameters.encryption_args.public_key = e.target.value; + this.props.setParameters(this.props.parameters); + }} /> + + } - {this.props.parameters.encryption_scheme == "Password" && <> + { this.props.parameters.encryption_scheme === "Password" && {T("Password")} @@ -132,7 +133,7 @@ class OfflineCollectorParameters extends React.Component { this.props.setParameters(this.props.parameters); }} /> - + }