Port CI testing from Appveyor to GitHub Actions. (Velocidex#1) (Veloc…

…idex#512)

.gitattributes

@@ -0,0 +1,18 @@
+* text=auto
+
+*.go text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf
+*.c text eol=lf
+*.h text eol=lf
+*.md text eol=lf
+*.golden text eol=lf
+*.txt text eol=lf
+*.html text eol=lf
+*.js text eol=lf
+*.css text eol=lf
+
+*.db binary
+*.json binary
+*.csv binary
+*.zip binary

.github/workflows/go.yml

@@ -1,18 +1,16 @@
-name: Go
-on: [push]
-jobs:
+# Only trigger when a PR is committed.
+name: Linux Build All Arches
+on:
+  push:
+    branches:
+      - main
 
+jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.14
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.14
-      id: go
-
     - name: Check out code into the Go module directory
       uses: actions/checkout@v1
 
@@ -26,34 +24,33 @@ jobs:
       with:
         node-version: 12
 
+    - name: Cache node-modules
+      uses: actions/cache@v2
+      with:
+        path: |
+          **/node_modules
+        key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+
     - name: npm install gui
       run: |
             cd gui/static/
             npm install
             node node_modules/gulp/bin/gulp.js compile
             cd ../../
 
-    - name: Build
+    - name: Build All Architectures
+      # Uncomment the architectures you want here. NOTE: DarwinBase
+      # does not include yara or modules with C compilers needed.
       run: |
         mkdir ./output/
         export PATH=$PATH:~/go/bin/
         go run make.go -v Linux
-
-    - name: Test
-      run: |
-        go test ./...
-        ./output/velociraptor*linux* golden ./artifacts/testdata./server./testcases/ --env srcDir=./
-
-    - name: Build All Architectures
-      run: |
-        export PATH=$PATH:~/go/bin/
         go run make.go -v Windows
-        go run make.go -v Windowsx86
+        echo skipping go run make.go -v Windowsx86
         go run make.go -v DarwinBase
-        zip Binaries.zip output/*
 
     - name: StoreBinaries
       uses: actions/upload-artifact@v1
       with:
         name: Binaries.zip
-        path: Binaries.zip
+        path: output

.github/workflows/windows.yml

@@ -0,0 +1,118 @@
+name: Windows Test
+on: [push, pull_request]
+jobs:
+  build:
+    name: Windows Test
+    runs-on: windows-latest
+    steps:
+    - name: Set up Go 1.14
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: setup env
+      run: |
+        echo "::set-env name=GOPATH::$(dirname $GITHUB_WORKSPACE)"
+        echo "::add-path::$(dirname $GITHUB_WORKSPACE)/bin"
+      shell: bash
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Configure test environment
+      shell: cmd
+      if: always()
+      run: |
+        echo %PATH%
+        echo %GOPATH%
+        mklink c:\Users\link c:\Windows
+        mkdir "C:\Program Files\Velociraptor"
+        mkdir c:\tmp
+        echo Hello > C:\hello.txt
+        echo HelloADS > C:\hello.txt:myads
+        fsutil file setshortname C:\hello.txt hi.txt
+        sc.exe create TestingDetection1 binPath="%COMSPEC% /Q /c echo 'COMSPEC testing 1"
+        echo "VSStest" > c:\Users\test.txt
+        echo "VSStest2" > c:\Users\test2.txt
+        regedit /S artifacts/testdata/windows/init.reg
+
+    - name: Build
+      if: always()
+      env:
+        CC: x86_64-w64-mingw32-gcc
+      shell: bash
+      run: |
+        mkdir gui/static/dist/
+        echo > gui/static/dist/third-party.bundle.js
+        echo > gui/static/dist/grr-ui.bundle.js
+        go run make.go -v WindowsTest
+
+    - name: Prepare second stage
+      if: always()
+      shell: cmd
+      # We have to wait a short time between the service creation
+      # event to be flushed to disk. Hopefully building the test
+      # binary takes long enough.
+
+      # Creating a second service - in the end we have
+      # TestingDetection2 in current logs, TestingDetection1 in
+      # previously cleared logs (but in VSS).
+      run: |
+        vssadmin create shadow /for=c:
+        echo "VSStest2 with more data" > c:\Users\test2.txt
+        echo Clearing the event logs
+        wevtutil.exe cl System
+
+        echo Create second service.
+        sc.exe create TestingDetection2 binPath="%COMSPEC% /Q /c echo 'COMSPEC testing 2"
+
+    - name: Test
+      shell: bash
+      if: always()
+      env:
+        # Disable CGO for building tests - it takes too long and it is
+        # not needed (mainly disables Yara building again).
+        CGO_ENABLED: "0"
+
+      run: |
+        go test -v ./...
+
+    - name: Test Golden Generic
+      shell: cmd
+      if: always()
+      # We depend on the second service logs to be flushed to disk -
+      # hopefulling the unit tests take long enough for this to
+      # happen.
+
+      # Create a second VSS right before testing - in most cases the
+      # second VSS is identical to C: and VSS artifacts should
+      # **always** prefer to emit C: based locations over VSS paths.
+      run: |
+        vssadmin create shadow /for=c:
+
+        echo Running OS generic tests.
+        output\velociraptor.exe -v golden D:\a\velociraptor\velociraptor\artifacts\testdata\server\testcases\ --env srcDir=d:\a\velociraptor\velociraptor\ --config D:\a\velociraptor\velociraptor\artifacts\testdata\windows\github_actions.config.yaml
+
+    - name: Test Golden Windows
+      shell: cmd
+      if: always()
+      run: |
+        echo Running windows specific tests.
+        output\velociraptor.exe -v golden D:\a\velociraptor\velociraptor\artifacts\testdata\windows\ --env srcDir=d:\a\velociraptor\velociraptor\ --config D:\a\velociraptor\velociraptor\artifacts\testdata\windows\github_actions.config.yaml
+
+    - name: Upload Build Artifacts
+      if: always()
+      shell: bash
+      run: |
+        mkdir -p artifact_output/windows/
+        cp artifacts/testdata/windows/*.out* artifact_output/windows/
+
+        mkdir -p artifact_output/server/
+        cp artifacts/testdata/server/testcases/*.out* artifact_output/server/
+
+    - uses: actions/upload-artifact@master
+      if: always()
+      with:
+        name: artifact
+        path: artifact_output

README.md

@@ -125,15 +125,23 @@ We have a pretty frequent release schedule but if you see a new
 feature submitted that you are really interested in, we would love to
 have more testing prior to the official release.
 
-We have a CI pipeline at https://www.velocidex.com/ci managed by
-AppVeyor. The pipeline produces a complete windows binary
-built at each commit poiint - simply click on the `artifacts` tab,
-scroll down and download `velociraptor.exe`
-
-Additionally we build all binaries using Github actions. Simply click
-the [actions tab](https://github.com/Velocidex/velociraptor/actions)
-on Github and download the `Binaries.zip` file. This will contain
-binaries for Windows, MacOS and Linux.
+We have a CI pipeline managed by GitHub actions. You can see the
+pipeline by clicking the actions tab on our GitHub project. There are
+two workflows:
+
+1. Windows Test: this workflow build a minimal version of the
+   Velociraptor binary (without the GUI) and runs all the tests on
+   it. We also test various windows support functions in this
+   pipeline. This pipeline builds on every push in each PR.
+
+2. Linux Build All Arches: This pipeline builds complete binaries for
+   many supported architectures. It only runs when the PR is merged
+   into the master branch.
+
+If you fork the project on GitHub, the pipelines will run on your own
+fork as well. If you need to prepare a PR for a new feature or modify
+an existing feature you can use this to build your own binaries for
+testing on all architectures.
 
 
 ## Getting help

api/builder.go

@@ -122,6 +122,12 @@ func startSharedSelfSignedFrontend(
 		return err
 	}
 
+	// Combine both frontend and GUI on HTTP server.
+	if config_obj.GUI.UsePlainHttp && config_obj.Frontend.UsePlainHttp {
+		return StartFrontendPlainHttp(
+			ctx, wg, config_obj, server_obj, mux)
+	}
+
 	return StartFrontendHttps(ctx, wg,
 		config_obj, server_obj, router)
 }
@@ -144,7 +150,11 @@ func startSelfSignedFrontend(
 		}
 
 		// Start the GUI separately on a different port.
-		err = StartSelfSignedGUI(ctx, wg, config_obj, router)
+		if config_obj.GUI.UsePlainHttp {
+			err = StartHTTPGUI(ctx, wg, config_obj, router)
+		} else {
+			err = StartSelfSignedGUI(ctx, wg, config_obj, router)
+		}
 		if err != nil {
 			return err
 		}
@@ -394,6 +404,63 @@ func StartFrontendWithAutocert(
 	return nil
 }
 
+func StartHTTPGUI(
+	ctx context.Context,
+	wg *sync.WaitGroup,
+	config_obj *config_proto.Config, mux http.Handler) error {
+	logger := logging.Manager.GetLogger(config_obj, &logging.GUIComponent)
+
+	listenAddr := fmt.Sprintf("%s:%d",
+		config_obj.GUI.BindAddress,
+		config_obj.GUI.BindPort)
+
+	server := &http.Server{
+		Addr:     listenAddr,
+		Handler:  mux,
+		ErrorLog: logging.NewPlainLogger(config_obj, &logging.FrontendComponent),
+
+		// https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
+		ReadTimeout:  500 * time.Second,
+		WriteTimeout: 900 * time.Second,
+		IdleTimeout:  15 * time.Second,
+	}
+
+	logger.WithFields(
+		logrus.Fields{
+			"listenAddr": listenAddr,
+		}).Info("GUI is ready to handle HTTP requests")
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		err := server.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			logger.Error("GUI Server error", err)
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-ctx.Done()
+
+		logger.Info("Stopping GUI Server")
+		timeout_ctx, cancel := context.WithTimeout(
+			context.Background(), 10*time.Second)
+		defer cancel()
+
+		server.SetKeepAlivesEnabled(false)
+		err := server.Shutdown(timeout_ctx)
+		if err != nil {
+			logger.Error("GUI shutdown error ", err)
+		}
+		logger.Info("Shutdown GUI")
+	}()
+
+	return nil
+}
+
 func StartSelfSignedGUI(
 	ctx context.Context,
 	wg *sync.WaitGroup,