diff --git a/accessors/collector/collector.go b/accessors/collector/collector.go index 7518b456677..2044ebdd4a5 100644 --- a/accessors/collector/collector.go +++ b/accessors/collector/collector.go @@ -173,11 +173,10 @@ func (self *CollectorAccessor) maybeSetZipPassword( } // Check if data.zip exists at the top level. - if len(full_path.Components) > 0 { - return full_path, nil - } + root := full_path.Copy() + root.Components = nil - datazip := full_path.Dirname().Append("data.zip") + datazip := root.Append("data.zip") _, err := self.ZipFileSystemAccessor.LstatWithOSPath(datazip) if err != nil { // Nope - no data.zip so do not transform the pathspec. @@ -185,7 +184,7 @@ func (self *CollectorAccessor) maybeSetZipPassword( } // Check if metadata.json exists. If so, try to extract password - meta := full_path.Dirname().Append("metadata.json") + meta := root.Append("metadata.json") mhandle, err := self.ZipFileSystemAccessor.OpenWithOSPath(meta) if err != nil { // No metadata file is found - this might be a plain diff --git a/accessors/collector/collector_test.go b/accessors/collector/collector_test.go new file mode 100644 index 00000000000..e8b2b646cea --- /dev/null +++ b/accessors/collector/collector_test.go @@ -0,0 +1,119 @@ +package collector_test + +import ( + "path/filepath" + "testing" + + "github.com/Velocidex/ordereddict" + "github.com/sebdah/goldie/v2" + "github.com/stretchr/testify/suite" + "www.velocidex.com/golang/velociraptor/file_store/test_utils" + "www.velocidex.com/golang/velociraptor/json" + "www.velocidex.com/golang/velociraptor/logging" + "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/vql/acl_managers" + "www.velocidex.com/golang/velociraptor/vql/filesystem" + "www.velocidex.com/golang/vfilter" + + _ "www.velocidex.com/golang/velociraptor/accessors/file" + _ "www.velocidex.com/golang/velociraptor/accessors/ntfs" +) + +const ( + TestFrontendCertificate = `-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIQcyUFy1oMUr4O4sIOhom/jDANBgkqhkiG9w0BAQsFADAa +MRgwFgYDVQQKEw9WZWxvY2lyYXB0b3IgQ0EwIBcNMjMwNDEzMTgzMjUzWhgPMjEy +MzAzMjAxODMyNTNaMDQxFTATBgNVBAoTDFZlbG9jaXJhcHRvcjEbMBkGA1UEAxMS +VmVsb2NpcmFwdG9yU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA9MSMbrFjmZs9bnpkel4vTQIyf+6Bpg60ByC7d6WWfBwvHdF1Qnfn1JO3Xo6p +53I1jPoagt0cZCzd6nwJXJ/3pclprmIOEBSc20pg5E0A/kpwn+bBoPNSrMF7+2/t +DvXP0Lvs/1OqUMjF8pCs6vnSKigaptn+0Et3GpzWjwCghqPcJBOuEuPQmR3HyHfs +dsMooCjuYcRcS9MXioT97SSjxeug0oTXHaKCnQ7txoxuN2+nNdr03mUu07TOUbRp +X3NsiaoESl/9IDC/tz2XTBD3UxLze9pX9t4tdKEMK2+gdnrnioOw1D7WBoElECj9 ++89CRXlu3K15P1cNVB5htPzOgwIDAQABo38wfTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0j +BBgwFoAUO2IRSDwqgkZt5pkXdScs5BjoULEwHQYDVR0RBBYwFIISVmVsb2NpcmFw +dG9yU2VydmVyMA0GCSqGSIb3DQEBCwUAA4IBAQAhwcTMIdHqeR3FXOUREGjkjzC9 +vz+hPdXB6w9CMYDOAsmQojuo09h84xt7jD0iqs/K1WJpLSNV3FG5C0TQXa3PD1l3 +SsD5p4FfuqFACbPkm/oy+NA7E/0BZazC7iaZYjQw7a8FUx/P+eKo1S7z7Iq8HfmJ +yus5NlnoLmqb/3nZ7DyRWSo9HApmMdNjB6oJWrupSJajsw4Lsos2aJjkfzkg82W7 +aGSh9S6Icn1f78BAjJVLv1QBNlb+yGOhrcUWQHERPEpkb1oZJwkVVE1XCZ1C4tVj +PtlBbpcpPHB/R5elxfo+We6vmC8+8XBlNPFFp8LAAile4uQPVQjqy7k/MZ4W +-----END CERTIFICATE-----` + TestFrontendPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA9MSMbrFjmZs9bnpkel4vTQIyf+6Bpg60ByC7d6WWfBwvHdF1 +Qnfn1JO3Xo6p53I1jPoagt0cZCzd6nwJXJ/3pclprmIOEBSc20pg5E0A/kpwn+bB +oPNSrMF7+2/tDvXP0Lvs/1OqUMjF8pCs6vnSKigaptn+0Et3GpzWjwCghqPcJBOu +EuPQmR3HyHfsdsMooCjuYcRcS9MXioT97SSjxeug0oTXHaKCnQ7txoxuN2+nNdr0 +3mUu07TOUbRpX3NsiaoESl/9IDC/tz2XTBD3UxLze9pX9t4tdKEMK2+gdnrnioOw +1D7WBoElECj9+89CRXlu3K15P1cNVB5htPzOgwIDAQABAoIBAGAAy3gLOZ6hBgpU +FR7t3C2fRAFrogxozfHRw9Xc69ZIE67lXdGxSAvX2F9NI5T09c4Stt1HLoCYHH6B +Igbjc3XiNwI/0XY7L37PgItrLI2Q0vXUw3OGnJHH3gIz10472cPsQbuvrCi9Zu6K +ElijnewNCM8Sx+AZCWE1zO4P9+Z2kF9LvWzDwAa643jQ/Dg+S68zCFqjJCVJBGm+ +LQxDs6dbArvOiEbuZs2wDt0d1kZF+BRljUTMoCpdf3jmFj3f0Jc1AFaz1eHG9Gte +XIUpbWmV2ATABSW2kDkVdXx+m/w1r9PZCLLfq54fIOlm2IeAiM3rDmM4ZSTUYEPn +mJP03xECgYEA+jS7DiS3bB/MeD+5qsgS07qJhOrX17s/SlamC1dQqz+koJLl98JX +CqyafFmdSz7PK2S2+OOazngwx26Kc3MZFoD9IQ2tuWmwDgbY8EQs5Cs37By2YRZJ +DdjvVf48pCKiXxIhvFjW/5CTemNAAu4CXg5Lkp7UVVrOmf5BmjMmE0sCgYEA+m+U +QMF0f7KLM4MU81yAMJdG4Sq4s9i4RmXes2FOUd4UoG7vEpycMKkmEaqiUVmRHPjp +P6Dwq3CK+FVFMpCeWjn6KkxwpdWWO9lglI0npFcPNW/PzPOv4mSNtCAcpHrKFP0R +3jbc8UhgtFxDZoeUih7cO2iTO7kELBCeKUzw9qkCgYBgVYcj1e0tWzztm5OP9sKQ +9MRYAdei/zxKEfySZ0bu+G0ZShXzA8dhm71LXXGbdA5t5bQxNej3z/zv/FagRtOE +/5r2a/7UYaXgcLB8KbOjEiTQ6ukpjlwIUdssn9uXUqJzulZ03zvAYFj4CVivCBav +Qg/E3xRf3LupPOTjSwhA6wKBgQDAH3tnlkHueSWLNiOLc0owfM12jhS2fCsabqpD +iQHRkoLWdWRZLeYw+oLnCLWPnRvTUy11j90yWJt0Wc5FNWcWJuZBLvU4c7vWXDRY +olVoIRXc09NiEwy6rJN9PSlcEYsYQPFFPWeQfwsZMrLOZHLS50vjE53oMk7+Ex2S +56DwSQKBgQC+iHbsbxloZjVMy01V21Sh9RwIpYrodEmwlTZf2jzaYloPadHu4MX1 +jHG+zzeC/EJ3wFOKTSJ/Tmjo6N3Xaq9V7WeL8eBdtBtPztqN1yveTt94mZZ+fuID +BhI8P2RbNR2Yey5nnhFQcoTxpmVw3EYwE01nkxoPJRs/QVvxi9Mepg== +-----END RSA PRIVATE KEY-----` +) + +type TestSuite struct { + test_utils.TestSuite +} + +func (self *TestSuite) SetupTest() { + self.ConfigObj = self.LoadConfig() + self.ConfigObj.Frontend.Certificate = TestFrontendCertificate + self.ConfigObj.Frontend.PrivateKey = TestFrontendPrivateKey + + self.TestSuite.SetupTest() +} + +func (self *TestSuite) TestAutomaticDecryption() { + manager, _ := services.GetRepositoryManager(self.ConfigObj) + + builder := services.ScopeBuilder{ + Config: self.ConfigObj, + ACLManager: acl_managers.NullACLManager{}, + Logger: logging.NewPlainLogger(self.ConfigObj, &logging.FrontendComponent), + Env: ordereddict.NewDict(), + } + + scope := manager.BuildScope(builder) + + fixture_path, _ := filepath.Abs( + "../../vql/tools/collector/fixtures/offline_encrypted.zip") + + root_path_spec := (filesystem.PathSpecFunction{}).Call(self.Ctx, scope, + ordereddict.NewDict().Set("DelegatePath", fixture_path)) + + lines := []vfilter.Row{} + for row := range (filesystem.GlobPlugin{}).Call(self.Ctx, + scope, ordereddict.NewDict(). + Set("globs", "**"). + Set("accessor", "collector"). + Set("root", root_path_spec)) { + + full_path, _ := scope.Associative(row, "OSPath") + lines = append(lines, full_path) + } + + g := goldie.New(self.T()) + g.Assert(self.T(), "TestAutomaticDecryption", json.MustMarshalIndent(lines)) +} + +func TestCollectorAccessor(t *testing.T) { + suite.Run(t, &TestSuite{}) +} diff --git a/accessors/collector/testdata/TestAutomaticDecryption.golden b/accessors/collector/testdata/TestAutomaticDecryption.golden new file mode 100644 index 00000000000..4c64bf03d80 --- /dev/null +++ b/accessors/collector/testdata/TestAutomaticDecryption.golden @@ -0,0 +1,9 @@ +[ + "/collection_context.json", + "/log.json", + "/log.json.index", + "/requests.json", + "/results", + "/results/Demo.Plugins.GUI.json", + "/results/Demo.Plugins.GUI.json.index" +] \ No newline at end of file diff --git a/api/download.go b/api/download.go index 4f923a68f3c..a4c1a982524 100644 --- a/api/download.go +++ b/api/download.go @@ -125,6 +125,11 @@ func vfsFileDownloadHandler() http.Handler { if org_id == "" { org_id = authenticators.GetOrgIdFromRequest(r) } + + org_id = utils.NormalizedOrgId(org_id) + + utils.Debug(org_id) + org_manager, err := services.GetOrgManager() if err != nil { returnError(w, 404, err.Error()) diff --git a/vql/tools/collector/collector_test.go b/vql/tools/collector/collector_test.go index 01215982b5e..3cf2721daa5 100644 --- a/vql/tools/collector/collector_test.go +++ b/vql/tools/collector/collector_test.go @@ -199,6 +199,8 @@ func (self *TestSuite) SetupTest() { self.ConfigObj.Services.ServerArtifacts = true self.LoadArtifactsIntoConfig([]string{customCollectionWithTypes}) self.LoadArtifactsIntoConfig(importHuntArtifacts) + self.ConfigObj.Frontend.Certificate = TestFrontendCertificate + self.ConfigObj.Frontend.PrivateKey = TestFrontendPrivateKey self.TestSuite.SetupTest() diff --git a/vql/tools/collector/fixtures/offline_encrypted.zip b/vql/tools/collector/fixtures/offline_encrypted.zip new file mode 100755 index 00000000000..3795403b229 Binary files /dev/null and b/vql/tools/collector/fixtures/offline_encrypted.zip differ diff --git a/vql/tools/collector/import_test.go b/vql/tools/collector/import_test.go index 1d79d87ee9c..d28e5c42b0f 100644 --- a/vql/tools/collector/import_test.go +++ b/vql/tools/collector/import_test.go @@ -28,6 +28,56 @@ import ( _ "www.velocidex.com/golang/velociraptor/vql/protocols" ) +const ( + TestFrontendCertificate = `-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIQcyUFy1oMUr4O4sIOhom/jDANBgkqhkiG9w0BAQsFADAa +MRgwFgYDVQQKEw9WZWxvY2lyYXB0b3IgQ0EwIBcNMjMwNDEzMTgzMjUzWhgPMjEy +MzAzMjAxODMyNTNaMDQxFTATBgNVBAoTDFZlbG9jaXJhcHRvcjEbMBkGA1UEAxMS +VmVsb2NpcmFwdG9yU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA9MSMbrFjmZs9bnpkel4vTQIyf+6Bpg60ByC7d6WWfBwvHdF1Qnfn1JO3Xo6p +53I1jPoagt0cZCzd6nwJXJ/3pclprmIOEBSc20pg5E0A/kpwn+bBoPNSrMF7+2/t +DvXP0Lvs/1OqUMjF8pCs6vnSKigaptn+0Et3GpzWjwCghqPcJBOuEuPQmR3HyHfs +dsMooCjuYcRcS9MXioT97SSjxeug0oTXHaKCnQ7txoxuN2+nNdr03mUu07TOUbRp +X3NsiaoESl/9IDC/tz2XTBD3UxLze9pX9t4tdKEMK2+gdnrnioOw1D7WBoElECj9 ++89CRXlu3K15P1cNVB5htPzOgwIDAQABo38wfTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0j +BBgwFoAUO2IRSDwqgkZt5pkXdScs5BjoULEwHQYDVR0RBBYwFIISVmVsb2NpcmFw +dG9yU2VydmVyMA0GCSqGSIb3DQEBCwUAA4IBAQAhwcTMIdHqeR3FXOUREGjkjzC9 +vz+hPdXB6w9CMYDOAsmQojuo09h84xt7jD0iqs/K1WJpLSNV3FG5C0TQXa3PD1l3 +SsD5p4FfuqFACbPkm/oy+NA7E/0BZazC7iaZYjQw7a8FUx/P+eKo1S7z7Iq8HfmJ +yus5NlnoLmqb/3nZ7DyRWSo9HApmMdNjB6oJWrupSJajsw4Lsos2aJjkfzkg82W7 +aGSh9S6Icn1f78BAjJVLv1QBNlb+yGOhrcUWQHERPEpkb1oZJwkVVE1XCZ1C4tVj +PtlBbpcpPHB/R5elxfo+We6vmC8+8XBlNPFFp8LAAile4uQPVQjqy7k/MZ4W +-----END CERTIFICATE-----` + TestFrontendPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA9MSMbrFjmZs9bnpkel4vTQIyf+6Bpg60ByC7d6WWfBwvHdF1 +Qnfn1JO3Xo6p53I1jPoagt0cZCzd6nwJXJ/3pclprmIOEBSc20pg5E0A/kpwn+bB +oPNSrMF7+2/tDvXP0Lvs/1OqUMjF8pCs6vnSKigaptn+0Et3GpzWjwCghqPcJBOu +EuPQmR3HyHfsdsMooCjuYcRcS9MXioT97SSjxeug0oTXHaKCnQ7txoxuN2+nNdr0 +3mUu07TOUbRpX3NsiaoESl/9IDC/tz2XTBD3UxLze9pX9t4tdKEMK2+gdnrnioOw +1D7WBoElECj9+89CRXlu3K15P1cNVB5htPzOgwIDAQABAoIBAGAAy3gLOZ6hBgpU +FR7t3C2fRAFrogxozfHRw9Xc69ZIE67lXdGxSAvX2F9NI5T09c4Stt1HLoCYHH6B +Igbjc3XiNwI/0XY7L37PgItrLI2Q0vXUw3OGnJHH3gIz10472cPsQbuvrCi9Zu6K +ElijnewNCM8Sx+AZCWE1zO4P9+Z2kF9LvWzDwAa643jQ/Dg+S68zCFqjJCVJBGm+ +LQxDs6dbArvOiEbuZs2wDt0d1kZF+BRljUTMoCpdf3jmFj3f0Jc1AFaz1eHG9Gte +XIUpbWmV2ATABSW2kDkVdXx+m/w1r9PZCLLfq54fIOlm2IeAiM3rDmM4ZSTUYEPn +mJP03xECgYEA+jS7DiS3bB/MeD+5qsgS07qJhOrX17s/SlamC1dQqz+koJLl98JX +CqyafFmdSz7PK2S2+OOazngwx26Kc3MZFoD9IQ2tuWmwDgbY8EQs5Cs37By2YRZJ +DdjvVf48pCKiXxIhvFjW/5CTemNAAu4CXg5Lkp7UVVrOmf5BmjMmE0sCgYEA+m+U +QMF0f7KLM4MU81yAMJdG4Sq4s9i4RmXes2FOUd4UoG7vEpycMKkmEaqiUVmRHPjp +P6Dwq3CK+FVFMpCeWjn6KkxwpdWWO9lglI0npFcPNW/PzPOv4mSNtCAcpHrKFP0R +3jbc8UhgtFxDZoeUih7cO2iTO7kELBCeKUzw9qkCgYBgVYcj1e0tWzztm5OP9sKQ +9MRYAdei/zxKEfySZ0bu+G0ZShXzA8dhm71LXXGbdA5t5bQxNej3z/zv/FagRtOE +/5r2a/7UYaXgcLB8KbOjEiTQ6ukpjlwIUdssn9uXUqJzulZ03zvAYFj4CVivCBav +Qg/E3xRf3LupPOTjSwhA6wKBgQDAH3tnlkHueSWLNiOLc0owfM12jhS2fCsabqpD +iQHRkoLWdWRZLeYw+oLnCLWPnRvTUy11j90yWJt0Wc5FNWcWJuZBLvU4c7vWXDRY +olVoIRXc09NiEwy6rJN9PSlcEYsYQPFFPWeQfwsZMrLOZHLS50vjE53oMk7+Ex2S +56DwSQKBgQC+iHbsbxloZjVMy01V21Sh9RwIpYrodEmwlTZf2jzaYloPadHu4MX1 +jHG+zzeC/EJ3wFOKTSJ/Tmjo6N3Xaq9V7WeL8eBdtBtPztqN1yveTt94mZZ+fuID +BhI8P2RbNR2Yey5nnhFQcoTxpmVw3EYwE01nkxoPJRs/QVvxi9Mepg== +-----END RSA PRIVATE KEY-----` +) + func (self *TestSuite) TestCreateAndImportCollection() { closer := utils.MockTime(utils.NewMockClock(time.Unix(10, 10))) defer closer() @@ -180,3 +230,34 @@ func (self *TestSuite) TestImportCollectionFromFixture() { // The new flow was created on the same client id as before. assert.Equal(self.T(), context2.ClientId, context.ClientId) } + +func (self *TestSuite) TestImportX509CollectionFromFixture() { + manager, _ := services.GetRepositoryManager(self.ConfigObj) + + builder := services.ScopeBuilder{ + Config: self.ConfigObj, + ACLManager: acl_managers.NullACLManager{}, + Logger: logging.NewPlainLogger(self.ConfigObj, &logging.FrontendComponent), + Env: ordereddict.NewDict(), + } + + ctx := self.Ctx + scope := manager.BuildScope(builder) + + import_file_path, err := filepath.Abs("fixtures/offline_encrypted.zip") + assert.NoError(self.T(), err) + + result := collector.ImportCollectionFunction{}.Call(ctx, scope, + ordereddict.NewDict(). + Set("client_id", "auto"). + Set("hostname", "MyNewHost"). + Set("filename", import_file_path)) + context, ok := result.(*proto.ArtifactCollectorContext) + assert.True(self.T(), ok) + + assert.Equal(self.T(), []string{"Demo.Plugins.GUI"}, + context.ArtifactsWithResults) + assert.Equal(self.T(), uint64(1), context.TotalCollectedRows) + assert.Equal(self.T(), flows_proto.ArtifactCollectorContext_FINISHED, + context.State) +}