JIT compiler + bugfixes

.github/workflows/go.yml

@@ -1,4 +1,4 @@
-name: goloader Testing
+name: goloader
 
 on:
   push:
@@ -14,58 +14,36 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.8.x, 1.9.x, 1.10.x, 1.11.x, 1.12.x, 1.13.x, 1.14.x, 1.15.x, 1.16.x, 1.17.x, 1.18.x, 1.19.x]
-        os:  [ubuntu-latest, windows-latest, macos-latest]
-        exclude:
-          - os: macos-latest
-            go-version: 1.8.x
-          - os: macos-latest
-            go-version: 1.9.x
-          - os: macos-latest
-            go-version: 1.10.x
-        include:
-          - os: macos-11
-            go-version: 1.8.x
-          - os: macos-11
-            go-version: 1.9.x
-          - os: macos-11
-            go-version: 1.10.x
+        go-version: [ 1.18.X, 1.19.X, 1.20.X , 1.21.X ]
+        os:  [ubuntu-latest, windows-latest, macos-latest, [self-hosted, Linux, ARM64], [self-hosted, macOS, ARM64]]
+        cgo-enabled: ["CGO_ENABLED=0", "CGO_ENABLED=1"]
+        dynlink: ["JIT_GC_DYNLINK=1", "JIT_GC_DYNLINK=0"]
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-
+        check-latest: true
+        cache-dependency-path: jit/go.sum
+
     - name: Rename cmd/internal
       shell: sh
-      run:         
-        cp -r $GOROOT/src/cmd/internal $GOROOT/src/cmd/objfile
-
+      run: |
+        eval $(go env | sed -r 's/^(set )?(\w+)=("|'"'"'?)(.*)\3$/\2="\4"/g') && cp -r $GOROOT/src/cmd/internal $GOROOT/src/cmd/objfile
+
+    - name: Patch gc
+      shell: sh
+      run:
+        go run -a github.com/eh-steve/goloader/jit/patchgc@2aedbae4eb66fb4d0a4f24efbeb1873007792bca
+
     - name: Checkout code
       uses: actions/checkout@v3
       with:
         path: ${{ env.GOPATH }}/src/github.com/${{ github.repository }}
-
-    - name: Build
-      run:
-        go build github.com/pkujhd/goloader/examples/loader
-
-    - name: Compile const.go
-      shell: sh
-      run:
-        go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/const/const.go
-
-    - name: Compile base.go
+
+    - name: Test
       shell: sh
       run:
-        go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/base/base.go
-
-    - name: Test base.o
-      run: 
-        ./loader -o base.o -run main.main
-
-    - name: Test const.o
-      run:
-        ./loader -o const.o -run main.main
+        cd $GOPATH/src/github.com/eh-steve/goloader/jit && export ${{ matrix.cgo-enabled }} ${{ matrix.dynlink }} && go test -a -c . && ./jit.test -test.v

.travis.yml

@@ -28,8 +28,8 @@ before_install:
 
 script:
   - export GO111MODULE=auto
-  - go build github.com/pkujhd/goloader/examples/loader
-  - go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/base/base.go
-  - go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/const/const.go
+  - go build github.com/eh-steve/goloader/examples/loader
+  - go tool compile $GOPATH/src/github.com/eh-steve/goloader/examples/base/base.go
+  - go tool compile $GOPATH/src/github.com/eh-steve/goloader/examples/const/const.go
   - ./loader -o base.o -run main.main
   - ./loader -o const.o -run main.main

README.md

@@ -1,70 +1,170 @@
+# Goloader/JIT Compiler for Go
 
-# Goloader
+[![Build Status](https://github.com/eh-steve/goloader/actions/workflows/go.yml/badge.svg)](https://github.com/eh-steve/goloader/actions/workflows/go.yml)
 
-![Build Status](https://github.com/pkujhd/goloader/workflows/goloader%20Testing/badge.svg)
+The `goloader/jit` package can compile and load Go code from text, file, folder or remote package (including code with
+package imports).
 
-Goloader can load and run Golang code at runtime.
+It automatically resolves package dependencies recursively, and provides a type safe way of interacting with the built
+functions.
 
-Forked from **https://github.com/dearplain/goloader**, Take over maintenance because the original author is not in maintenance
+Forked from [dearplain](https://github.com/dearplain/goloader) and [pkujhd](https://github.com/pkujhd/goloader).
 
-## How does it work?
-
-Goloader works like a linker: it relocates the address of symbols in an object file, generates runnable code, and then reuses the runtime function and the type pointer of the loader.
+# Usage
 
-Goloader provides some information to the runtime and gc of Go, which allows it to work correctly with them.
+## Build
 
-Please note that Goloader is not a scripting engine. It reads the output of Go compiler and makes them runnable. All features of Go are supported, and run just as fast and lightweight as native Go code.
+**Make sure you're using go >= 1.18.**
 
-## Comparison with plugin
+First, execute the following command. This is because Goloader relies on the internal package, which is forbidden by the
+Go compiler.
 
-Goloader reuses the Go runtime, which makes it much smaller. And code loaded by Goloader is unloadable.
+```
+cp -r $GOROOT/src/cmd/internal $GOROOT/src/cmd/objfile
+```
 
-Goloader supports pprof tool(Yes, you can see code loaded by Goloader in pprof).
+## Go compiler patch
 
-## Build
+To allow the loader to know the types of exported functions, this package will attempt to patch the Go compiler (gc) to
+emit these if not already patched.
 
-**Make sure you're using go >= 1.8.**
+The effect of the patch can be found in [`jit/gc.patch`](https://github.com/eh-steve/goloader/blob/master/jit/gc.patch).
 
-First, execute the following command, then do build and test. This is because Goloader relies on the internal package, which is forbidden by the Go compiler.
-```
-cp -r $GOROOT/src/cmd/internal $GOROOT/src/cmd/objfile
+```bash
+go install github.com/eh-steve/goloader/jit/patchgc@latest
+# You may need to run patchgc as sudo if your $GOROOT is owned by root
+# (alternatively `chown -R $USER:$USER $GOROOT`)
+patchgc
 ```
 
-## Examples
+## Example Usage
+
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/eh-steve/goloader/jit"
+)
+
+func main() {
+	conf := jit.BuildConfig{
+		KeepTempFiles:   false,          // Files are copied/written to a temp dir to ensure it is writable. This retains the temporary copies
+		ExtraBuildFlags: []string{"-x"}, // Flags passed to go build command
+		BuildEnv:        nil,            // Env vars to set for go build toolchain
+		TmpDir:          "",             // To control where temporary files are copied
+		DebugLog:        true,           //
+	}
+
+	loadable, err := jit.BuildGoFiles(conf, "./path/to/file1.go", "/path/to/file2.go")
+	if err != nil {
+		panic(err)
+	}
+	// or
+	loadable, err = jit.BuildGoPackage(conf, "./path/to/package")
+	if err != nil {
+		panic(err)
+	}
+	// or
+	loadable, err = jit.BuildGoPackageRemote(conf, "github.com/some/package/v4", "latest")
+	if err != nil {
+		panic(err)
+	}
+	// or
+	loadable, err = jit.BuildGoText(conf, `
+package mypackage
+
+import "encoding/json"
+
+func MyFunc(input []byte) (interface{}, error) {
+	var output interface{}
+	err := json.Unmarshal(input, &output)
+	return output, err
+}
+`)
+
+	if err != nil {
+		panic(err)
+	}
+
+	module, err := loadable.Load()
+	// module.SymbolsByPkg is a map[string]map[string]interface{} of all packages and their exported functions and global vars
+	symbols := module.SymbolsByPkg[loadable.ImportPath]
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		err = module.Unload()
+		if err != nil {
+			panic(err)
+		}
+	}()
+	switch f := symbols["MyFunc"].(type) {
+	case func([]byte) (interface{}, error):
+		result, err := f([]byte(`{"k":"v"}`))
+		if err != nil {
+			panic(err)
+		}
+		fmt.Println(result)
+	default:
+		panic("Function signature was not what was expected")
+	}
+}
 
 ```
-export GO111MODULE=auto
-go build github.com/pkujhd/goloader/examples/loader
 
-go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/schedule/schedule.go
-./loader -o schedule.o -run main.main -times 10
+## How does it work?
 
-go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/base/base.go
-./loader -o base.o -run main.main
+Goloader works like a linker, it relocates the addresses of symbols in an object file, generates runnable code, and then
+reuses the runtime functions and the type pointers of the loader where available.
 
-go tool compile $GOPATH/src/github.com/pkujhd/goloader/examples/http/http.go
-./loader -o http.o -run main.main
+Goloader provides some information to the runtime and garbage collector of Go, which allows it to work correctly with
+them.
 
-go install github.com/pkujhd/goloader/examples/basecontext
-go tool compile -I $GOPATH/pkg/`go env GOOS`_`go env GOARCH`/ $GOPATH/src/github.com/pkujhd/goloader/examples/inter/inter.go
-./loader -o $GOPATH/pkg/`go env GOOS`_`go env GOARCH`/github.com/pkujhd/goloader/examples/basecontext.a:github.com/pkujhd/goloader/examples/basecontext -o inter.o
+Please note that Goloader is not a scripting engine. It reads the archives emitted from the Go compiler and makes them
+runnable. All features of Go are supported, and run just as fast and lightweight as native Go code.
 
-#build multiple go files
-go tool compile -I $GOPATH/pkg/`go env GOOS`_`go env GOARCH`/ -o test.o test1.go test2.go
-./loader -o test.o -run main.main
+## Comparison with `plugin`
 
-```
+Plugin:
 
-## Warning
+* Can't load plugins not built with exact same versions of packages that host binary
+  uses (`plugin was built with a different version of package`) - this makes them basically unusable in most large
+  projects
+* Introduces dependency on `libdl`/CGo (and doesn't work on Windows)
+* Prevents linker deadcode elimination for unreachable methods (increases host binary size with unused methods)
+* Can't be unloaded/dynamically updated
+* Duplicates a lot of the go runtime (large binary sizes)
 
-Don't use "-s -w" compile argument, It strips symbol table.
+Goloader:
 
-This has currently only been tested and developed on:
+* Can build/load any packages (somewhat unsafely - it attempts to verify that types across JIT packages and host
+  packages match, but doesn't do the same checks for function signatures)
+* Pure Go - no dependency on `libdl`/Cgo
+* Patches host itabs containing unreachable methods instead of preventing linker deadcode elimination
+* Can be unloaded, and objects from one version of a JIT package can be converted at runtime to those from another
+  version, to allow dynamic adjustment of functions/methods without losing state
+* Reuses the runtime from the host binary (much smaller binaries)
 
-Golang 1.8-1.19 (x64/x86, darwin, linux, windows)
+Goloader supports pprof tool (yes, you can see code loaded by Goloader in pprof), but does not (yet) support debugging
+with `delve`.
 
-Golang 1.10-1.19 (arm, linux, android)
+## OS/Arch Compatibility
 
-Golang 1.8-1.19 (arm64, linux, android)
+JIT compiler tested/passing on:
 
-Golang 1.16-1.19 (arm64, darwin)
+| **OS/Arch**        | amd64/+CGo         | arm64/+CGo         | amd64/-CGo         | arm64/-CGo         |
+|--------------------|--------------------|--------------------|--------------------|--------------------|
+| Linux/go-1.20.3    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Darwin/go-1.20.3   | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Windows/go-1.20.3  | :heavy_check_mark: | :interrobang:      | :heavy_check_mark: | :interrobang:      |
+| Linux/go-1.19.4    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Darwin/go-1.19.4   | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Windows/go-1.19.4  | :heavy_check_mark: | :interrobang:      | :heavy_check_mark: | :interrobang:      |
+| Linux/go-1.18.8    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Darwin/go-1.18.8   | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Windows/go-1.18.8  | :x:                | :interrobang:      | :heavy_check_mark: | :interrobang:      |
+
+## Warning
+
+Don't use "-s -w" compile argument, It strips symbol table.

asm_bytes.go

@@ -1,33 +1,56 @@
 package goloader
 
 const (
-	x86amd64MOVcode  byte = 0x8B
-	x86amd64LEAcode  byte = 0x8D
-	x86amd64CMPLcode byte = 0x83
+	x86amd64MOVcode   byte = 0x8B
+	x86amd64LEAcode   byte = 0x8D
+	x86amd64CMPLcode  byte = 0x83
+	x86amd64CALLcode  byte = 0xE8
+	x86amd64CALL2code byte = 0xFF
+	x86amd64JMPcode   byte = 0xE9
 )
 
 // arm/arm64
 var (
-	armcode   = []byte{0x04, 0xF0, 0x1F, 0xE5} //LDR PC, [PC, #-4]
-	arm64code = []byte{
-		0x49, 0x00, 0x00, 0x58, // LDR X9 [PC+8]
-		0x20, 0x01, 0x1F, 0xD6} // BR X9
-	arm64BLcode = []byte{0x00, 0x00, 0x00, 0x94} // BL [PC+0x0]
+	armcode       = []byte{0x04, 0xF0, 0x1F, 0xE5} //LDR PC, [PC, #-4]
+	arm64CALLCode = []byte{
+		// X16 and X17 are the IP0 and IP1 intra-procedure-call corruptible registers -
+		// since Go only uses them for the stack prologue and epilogue calculations,
+		// and we should already be clear of that by the time we hit a R_CALLARM64,
+		// so we should be able to safely use them for far jumps
+		0x51, 0x00, 0x00, 0x58, // LDR X17 [PC+8] - read 64 bit address from PC+8 into X17
+		0x20, 0x02, 0x1f, 0xd6, // BR  X17 - jump to address in X17
+	}
+	arm64Bcode   = []byte{0x00, 0x00, 0x00, 0x14} // B [PC+0x0]
+	arm64NopCode = []byte{0x1f, 0x20, 0x03, 0xd5}
+)
+
+const (
+	armLDRCode8Bytes  = uint32(0x58000040) // LDR PC+8
+	armLDRCode12Bytes = uint32(0x58000060) // LDR PC+12
 )
 
 // x86/amd64
 var (
-	x86amd64JMPLcode        = []byte{0xff, 0x25, 0x00, 0x00, 0x00, 0x00} // JMPL *ADDRESS
+	x86amd64NOPcode      = byte(0x90)
+	x86amd64JMPLcode     = []byte{0xff, 0x25, 0x00, 0x00, 0x00, 0x00} // JMPL *ADDRESS
+	x86amd64JMPNearCode  = []byte{0xE9, 0x00, 0x00, 0x00, 0x00}       // JMP (PCREL offset)+4
+	x86amd64JMPShortCode = []byte{0xEB, 0x00}                         // JMP (PCREL offset)+1
+	x86amd64CALLFarCode  = []byte{
+		0xff, 0x15, 0x00, 0x00, 0x00, 0x00, // CALL ptr [RIP]
+	}
 	x86amd64replaceCMPLcode = []byte{
-		0x50,                                     // PUSH EAX
-		0x53,                                     // PUSH EBX
-		0x48, 0x8b, 0x05, 0x0f, 0x00, 0x00, 0x00, // MOVE EAX x
-		0x48, 0x8b, 0x18, // MOVE EBX [EAX]
-		0x48, 0x83, 0xfb, 0x00, // CMPL EBX x(8bits)
-		0x5b,                               // POP EBX
-		0x58,                               // POP EAX
-		0xff, 0x25, 0x08, 0x00, 0x00, 0x00} // JMPL *ADDRESS
+		0x50,                                                       // PUSH RAX
+		0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MOVABS RAX, imm64 (64 bit)
+		0x48, 0x83, 0x38, 0x00, // CMPL [RAX] x(8bits)
+		0x58, // POP RAX
+	}
+	x86amd64replaceMOVQcodeRAX = []byte{
+		0x48, 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MOVABS RAX, [addr64] (64 bit)
+	}
 	x86amd64replaceMOVQcode = []byte{
-		0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //MOVE RxX x
-		0xff, 0x25, 0x00, 0x00, 0x00, 0x00} //JMPL *ADDRESS
+		0x50,                                                       // PUSH RAX
+		0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MOVABS RAX, imm64 (64 bit)
+		0x48, 0x8b, 0x00, // MOV RxX, [RAX] (64 bit)
+		0x58, // POP RAX
+	}
 )