[refactor] modular redirectors

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: Release
 
 env:
   CARGO_TERM_COLOR: always
-  IMIX_CALLBACK_URI: http://127.0.0.1
+  IMIX_CALLBACK_URI: http://127.0.0.1:8000
 
 on:
   workflow_dispatch: ~

AGENTS.md

@@ -1,15 +1,21 @@
+# Agents
+
 Welcome to our repository! Most commands need to be run from the root of the project directory (e.g. where this `AGENT.md` file is located)
 
-# Project Structure
-  * `tavern/` includes our Golang server implementation, which hosts a GraphQL API, User-Interface (typescript), and a gRPC API used by agents.
-    * Our user interface is located in `tavern/internal/www` and we managed dependencies within that directory using `npm` 
-  * `implants/` contains Rust code that is deployed to target machines, such as our agent located in `implants/imix`.
+## Project Structure
+
+* `tavern/` includes our Golang server implementation, which hosts a GraphQL API, User-Interface (typescript), and a gRPC API used by agents.
+  * Our user interface is located in `tavern/internal/www` and we managed dependencies within that directory using `npm`
+* `implants/` contains Rust code that is deployed to target machines, such as our agent located in `implants/imix`.
+
+## Golang Tests
 
-# Golang Tests
 To run all Golang tests in our repository, please run `go test ./...` from the project root.
 
-# Code Generation
+## Code Generation
+
 This project heavily relies on generated code. When making changes to ent schemas, GraphQL, or user interface / front-end changes in the `tavern/internal/www/` directory, you will need to run `go generate ./...` from the project root directory to re-generate some critical files.
 
-# Additional Documentation
+## Additional Documentation
+
 Our user-facing documentation for the project is located in `docs/_docs` and can be referenced for additional information.

docs/_docs/admin-guide/tavern.md

@@ -110,7 +110,7 @@ By default Tavern only supports GRPC connections directly to the server. To Enab
 
 ### Webserver
 
-By default, Tavern will listen on `0.0.0.0:80`. If you ever wish to change this bind address then simply supply it to the `HTTP_LISTEN_ADDR` environment variable.
+By default, Tavern will listen on `0.0.0.0:8000`. If you ever wish to change this bind address then simply supply it to the `HTTP_LISTEN_ADDR` environment variable.
 
 ### Metrics
 
@@ -119,7 +119,7 @@ By default, Tavern does not export metrics. You may use the below environment co
 | Env Var | Description | Default | Required |
 | ------- | ----------- | ------- | -------- |
 | ENABLE_METRICS | Set to any value to enable the "/metrics" endpoint. | Disabled | No |
-| HTTP_METRICS_LISTEN_ADDR | Listen address for the metrics HTTP server, it must be different than the value of `HTTP_LISTEN_ADDR`. | `127.0.0.1:8080` | No |
+| HTTP_METRICS_LISTEN_ADDR | Listen address for the metrics HTTP server, it must be different than the value of `HTTP_LISTEN_ADDR`. | `127.0.0.1:8000` | No |
 
 ### Secrets
 
@@ -236,7 +236,7 @@ func main() {
  defer cancel()
 
  // Setup your Tavern URL (e.g. from env vars)
- tavernURL := "http://127.0.0.1"
+ tavernURL := "http://127.0.0.1:8000"
 
  // Configure Browser (uses the default system browser)
  browser := auth.BrowserFunc(browser.OpenURL)
@@ -280,7 +280,7 @@ Running Tavern with the `DISABLE_DEFAULT_TOMES` environment variable set will di
 
 ```sh
 DISABLE_DEFAULT_TOMES=1 go run ./tavern
-2024/03/02 01:32:22 [WARN] No value for 'HTTP_LISTEN_ADDR' provided, defaulting to 0.0.0.0:80
+2024/03/02 01:32:22 [WARN] No value for 'HTTP_LISTEN_ADDR' provided, defaulting to 0.0.0.0:8000
 2024/03/02 01:32:22 [WARN] MySQL is not configured, using SQLite
 2024/03/02 01:32:22 [WARN] OAuth is not configured, authentication disabled
 2024/03/02 01:32:22 [WARN] No value for 'DB_MAX_IDLE_CONNS' provided, defaulting to 10

docs/_docs/dev-guide/tavern.md

@@ -89,7 +89,7 @@ apt install -y graphviz
 ### Collect a Profile
 
 1. Start Tavern with profiling enabled: `ENABLE_PPROF=1 go run ./tavern`.
-2. Collect a Profile in desired format (e.g. png): `go tool pprof -png -seconds=10 http://127.0.0.1:80/debug/pprof/allocs?seconds=10 > .pprof/allocs.png`
+2. Collect a Profile in desired format (e.g. png): `go tool pprof -png -seconds=10 http://127.0.0.1:8000/debug/pprof/allocs?seconds=10 > .pprof/allocs.png`
     a. Replace "allocs" with the [name of the profile](https://pkg.go.dev/runtime/pprof#Profile) to collect.
     b. Replace the value of seconds with the amount of time you need to reproduce performance issues.
     c. Read more about the available profiling URL parameters [here](https://pkg.go.dev/net/http/pprof#hdr-Parameters).
@@ -113,27 +113,27 @@ The reverse shell system is designed to be highly scalable and resilient. It use
 
 The reverse shell system is composed of the following components:
 
-*   **gRPC Server**: The gRPC server is the entry point for the agent. It exposes the `ReverseShell` service, which is a bidirectional gRPC stream. The agent connects to this service to initiate a reverse shell session.
-*   **WebSocket Server**: The WebSocket server is the entry point for the user. It exposes a WebSocket endpoint that the user can connect to to interact with the reverse shell.
-*   **Pub/Sub Messaging System**: The pub/sub messaging system is the backbone of the reverse shell. It's used to decouple the gRPC server and the WebSocket server, and to provide a reliable and scalable way to transport messages between them. The system uses two topics:
-    *   **Input Topic**: The input topic is used to send messages from the user (via the WebSocket) to the agent (via the gRPC stream).
-    *   **Output Topic**: The output topic is used to send messages from the agent (via the gRPC stream) to the user (via the WebSocket).
-*   **Mux**: The `Mux` is a multiplexer that sits between the pub/sub system and the gRPC/WebSocket servers. It's responsible for routing messages between the two. There are two `Mux` instances:
-    *   **wsMux**: The `wsMux` is used by the WebSocket server. It subscribes to the output topic and publishes to the input topic.
-    *   **grpcMux**: The `grpcMux` is used by the gRPC server. It subscribes to the input topic and publishes to the output topic.
-*   **Stream**: A `Stream` represents a single reverse shell session. It's responsible for managing the connection between the `Mux` and the gRPC/WebSocket client.
-*   **sessionBuffer**: The `sessionBuffer` is used to order messages within a `Stream`. This is important because multiple users can be connected to the same shell session, and their messages need to be delivered in the correct order.
+* **gRPC Server**: The gRPC server is the entry point for the agent. It exposes the `ReverseShell` service, which is a bidirectional gRPC stream. The agent connects to this service to initiate a reverse shell session.
+* **WebSocket Server**: The WebSocket server is the entry point for the user. It exposes a WebSocket endpoint that the user can connect to to interact with the reverse shell.
+* **Pub/Sub Messaging System**: The pub/sub messaging system is the backbone of the reverse shell. It's used to decouple the gRPC server and the WebSocket server, and to provide a reliable and scalable way to transport messages between them. The system uses two topics:
+  * **Input Topic**: The input topic is used to send messages from the user (via the WebSocket) to the agent (via the gRPC stream).
+  * **Output Topic**: The output topic is used to send messages from the agent (via the gRPC stream) to the user (via the WebSocket).
+* **Mux**: The `Mux` is a multiplexer that sits between the pub/sub system and the gRPC/WebSocket servers. It's responsible for routing messages between the two. There are two `Mux` instances:
+  * **wsMux**: The `wsMux` is used by the WebSocket server. It subscribes to the output topic and publishes to the input topic.
+  * **grpcMux**: The `grpcMux` is used by the gRPC server. It subscribes to the input topic and publishes to the output topic.
+* **Stream**: A `Stream` represents a single reverse shell session. It's responsible for managing the connection between the `Mux` and the gRPC/WebSocket client.
+* **sessionBuffer**: The `sessionBuffer` is used to order messages within a `Stream`. This is important because multiple users can be connected to the same shell session, and their messages need to be delivered in the correct order.
 
 #### Communication Flow
 
-1.  The agent connects to the `ReverseShell` gRPC service.
-2.  The gRPC server creates a new `Shell` entity, a new `Stream`, and registers the `Stream` with the `grpcMux`.
-3.  The user connects to the WebSocket endpoint.
-4.  The WebSocket server creates a new `Stream` and registers it with the `wsMux`.
-5.  When the user sends a message, it's sent to the WebSocket server, which publishes it to the input topic via the `wsMux`.
-6.  The `grpcMux` receives the message from the input topic and sends it to the agent via the gRPC stream.
-7.  When the agent sends a message, it's sent to the gRPC server, which publishes it to the output topic via the `grpcMux`.
-8.  The `wsMux` receives the message from the output topic and sends it to the user via the WebSocket.
+1. The agent connects to the `ReverseShell` gRPC service.
+2. The gRPC server creates a new `Shell` entity, a new `Stream`, and registers the `Stream` with the `grpcMux`.
+3. The user connects to the WebSocket endpoint.
+4. The WebSocket server creates a new `Stream` and registers it with the `wsMux`.
+5. When the user sends a message, it's sent to the WebSocket server, which publishes it to the input topic via the `wsMux`.
+6. The `grpcMux` receives the message from the input topic and sends it to the agent via the gRPC stream.
+7. When the agent sends a message, it's sent to the gRPC server, which publishes it to the output topic via the `grpcMux`.
+8. The `wsMux` receives the message from the output topic and sends it to the user via the WebSocket.
 
 #### Distributed Architecture
 

docs/_docs/user-guide/getting-started.md

@@ -67,7 +67,7 @@ These configurations can be controlled via Environment Variables at `imix` compi
 
 ### Quests
 
-Now it's time to provide our [Beacon](/user-guide/terminology#beacon) it's first [Task](/user-guide/terminology#task). We do this, by creating a [Quest](/user-guide/terminology#quest) in the UI, which represents a collection of [Tasks](/user-guide/terminology#task) across one or more [Hosts](/user-guide/terminology#host). Let's open our UI, which should be available at [http://127.0.0.1:80/](http://127.0.0.1:80/).
+Now it's time to provide our [Beacon](/user-guide/terminology#beacon) it's first [Task](/user-guide/terminology#task). We do this, by creating a [Quest](/user-guide/terminology#quest) in the UI, which represents a collection of [Tasks](/user-guide/terminology#task) across one or more [Hosts](/user-guide/terminology#host). Let's open our UI, which should be available at [http://127.0.0.1:8000/](http://127.0.0.1:8000/).
 
 #### Beacon Selection
 

docs/_docs/user-guide/imix.md

@@ -15,8 +15,8 @@ Imix has compile-time configuration, that may be specified using environment var
 
 | Env Var | Description | Default | Required |
 | ------- | ----------- | ------- | -------- |
-| IMIX_CALLBACK_URI | URI for initial callbacks (must specify a scheme, e.g. `http://`) | `http://127.0.0.1:80` | No |
-| IMIX_SERVER_PUBKEY | The public key for the tavern server. | - | Yes |
+| IMIX_CALLBACK_URI | URI for initial callbacks (must specify a scheme, e.g. `http://`) | `http://127.0.0.1:8000` | No |
+| IMIX_SERVER_PUBKEY | The public key for the tavern server (obtain from server using `curl $IMIX_CALLBACK_URI/status`). | - | Yes |
 | IMIX_CALLBACK_INTERVAL | Duration between callbacks, in seconds. | `5` | No |
 | IMIX_RETRY_INTERVAL | Duration to wait before restarting the agent loop if an error occurs, in seconds. | `5` | No |
 | IMIX_PROXY_URI | Overide system settings for proxy URI over HTTP(S) (must specify a scheme, e.g. `https://`) | No proxy | No |
@@ -91,11 +91,12 @@ Building in the dev container limits variables that might cause issues and is th
 **Imix requires a server public key so it can encrypt messsages to and from the server check the server log for `level=INFO msg="public key: <SERVER_PUBKEY_B64>"`. This base64 encoded string should be passed to the agent using the environment variable `IMIX_SERVER_PUBKEY`**
 
 ## Optional build flags
+
 These flags are passed to cargo build Eg.:
 `cargo build --release --bin imix  --bin imix --target=x86_64-unknown-linux-musl --features foo-bar`
 
 - `--features grpc-doh` - Enable DNS over HTTP using cloudflare DNS for the grpc transport
-- `--features http --no-default-features` - Changes the default grpc transport to use HTTP/1.1. Requires running the http redirector.
+- `--features http1 --no-default-features` - Changes the default grpc transport to use HTTP/1.1. Requires running the http redirector.
 
 ### Linux
 

implants/lib/pb/src/config.rs

@@ -19,7 +19,7 @@ macro_rules! callback_uri {
     () => {
         match option_env!("IMIX_CALLBACK_URI") {
             Some(uri) => uri,
-            None => "http://127.0.0.1:80",
+            None => "http://127.0.0.1:8000",
         }
     };
 }
@@ -36,7 +36,7 @@ macro_rules! proxy_uri {
 
 /*
  * Compile-time constant for the agent callback URI, derived from the IMIX_CALLBACK_URI environment variable during compilation.
- * Defaults to "http://127.0.0.1:80/grpc" if this is unset.
+ * Defaults to "http://127.0.0.1:8000/grpc" if this is unset.
  */
 pub const CALLBACK_URI: &str = callback_uri!();
 

tavern/app.go

@@ -34,46 +34,82 @@ import (
 	"realm.pub/tavern/internal/graphql"
 	tavernhttp "realm.pub/tavern/internal/http"
 	"realm.pub/tavern/internal/http/stream"
-	"realm.pub/tavern/internal/redirector"
+	"realm.pub/tavern/internal/redirectors"
 	"realm.pub/tavern/internal/secrets"
 	"realm.pub/tavern/internal/www"
 	"realm.pub/tavern/tomes"
+
+	_ "realm.pub/tavern/internal/redirectors/http1"
 )
 
 func init() {
 	configureLogging()
 }
 
-func newApp(ctx context.Context, options ...func(*Config)) (app *cli.App) {
+func newApp(ctx context.Context) (app *cli.App) {
 	app = cli.NewApp()
 	app.Name = "tavern"
 	app.Description = "Teamserver implementation for Realm, see https://docs.realm.pub for more details"
 	app.Usage = "Time for an Adventure!"
 	app.Version = Version
-	app.Action = cli.ActionFunc(func(*cli.Context) error {
-		return run(ctx, options...)
-	})
+	app.Action = func(c *cli.Context) error {
+		return runTavern(
+			ctx,
+			ConfigureHTTPServerFromEnv(),
+			ConfigureMySQLFromEnv(),
+			ConfigureOAuthFromEnv("/oauth/authorize"),
+		)
+	}
 	app.Commands = []cli.Command{
 		{
-			Name: "redirector",
-			Usage: "Run a redirector connecting agents using a specific transport to the server",
-			Subcommands: []cli.Command{
-				{
-					Name: "http1",
-					Usage: "Run an HTTP/1.1 redirector",
-					Action: func(cCtx *cli.Context) error {
-						// Convert main.Config options to redirector.Config options
-						redirectorOptions := []func(*redirector.Config){
-							func(cfg *redirector.Config) {
-								// Apply main Config to get server settings
-								mainCfg := &Config{}
-								for _, opt := range options {
-									opt(mainCfg)
-								}
-								cfg.SetServer(mainCfg.srv)
-							},
+			Name:      "redirector",
+			Usage:     "Run a redirector connecting agents using a specific transport to the server",
+			ArgsUsage: "[upstream_address]",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "listen",
+					Usage: "Address to listen on for incoming redirector traffic (default: :8080)",
+					Value: ":8080",
+				},
+				cli.StringFlag{
+					Name:  "transport",
+					Usage: "Transport protocol to use for redirector (default: http1)",
+					Value: "http1",
+				},
+			},
+			Action: func(c *cli.Context) error {
+				var (
+					upstream  = c.Args().First()
+					listenOn  = c.String("listen")
+					transport = c.String("transport")
+				)
+				if upstream == "" {
+					return fmt.Errorf("gRPC upstream address is required (first argument)")
+				}
+				if listenOn == "" {
+					listenOn = ":8080"
+				}
+				if transport == "" {
+					transport = "http1"
+				}
+				slog.InfoContext(ctx, "starting redirector", "upstream", upstream, "transport", transport, "listen_on", listenOn)
+				return redirectors.Run(ctx, transport, listenOn, upstream)
+			},
+			Subcommands: cli.Commands{
+				cli.Command{
+					Name:  "list",
+					Usage: "List available redirectors",
+					Action: func(c *cli.Context) error {
+						redirectorNames := redirectors.List()
+						if len(redirectorNames) == 0 {
+							fmt.Println("No redirectors registered")
+							return nil
+						}
+						fmt.Println("Available redirectors:")
+						for _, name := range redirectorNames {
+							fmt.Printf("- %s\n", name)
 						}
-						return redirector.HTTPRedirectorRun(ctx, cCtx.Args().First(), redirectorOptions...)
+						return nil
 					},
 				},
 			},
@@ -82,8 +118,7 @@ func newApp(ctx context.Context, options ...func(*Config)) (app *cli.App) {
 	return
 }
 
-
-func run(ctx context.Context, options ...func(*Config)) error {
+func runTavern(ctx context.Context, options ...func(*Config)) error {
 	srv, err := NewServer(ctx, options...)
 	if err != nil {
 		return err
@@ -475,7 +510,6 @@ func newGRPCHandler(client *ent.Client, grpcShellMux *stream.Mux) http.Handler {
 			return
 		}
 
-
 		if contentType := r.Header.Get("Content-Type"); !strings.HasPrefix(contentType, "application/grpc") {
 			http.Error(w, "must specify Content-Type application/grpc", http.StatusBadRequest)
 			return

tavern/config.go

@@ -44,7 +44,7 @@ var (
 
 	// EnvHTTPListenAddr sets the address (ip:port) for tavern's HTTP server to bind to.
 	// EnvHTTPMetricsAddr sets the address (ip:port) for the HTTP metrics server to bind to.
-	EnvHTTPListenAddr        = EnvString{"HTTP_LISTEN_ADDR", "0.0.0.0:80"}
+	EnvHTTPListenAddr        = EnvString{"HTTP_LISTEN_ADDR", "0.0.0.0:8000"}
 	EnvHTTPMetricsListenAddr = EnvString{"HTTP_METRICS_LISTEN_ADDR", "127.0.0.1:8080"}
 
 	// EnvOAuthClientID set to configure OAuth Client ID.

tavern/internal/cryptocodec/cryptocodec.go

@@ -6,7 +6,6 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
-	"log"
 	"log/slog"
 	"runtime/debug"
 	"strconv"
@@ -23,8 +22,9 @@ var session_pub_keys = NewSyncMap()
 // This size limits the number of concurrent connections each server can handle.
 // I can't imagine a single server handling more than 10k connections at once but just in case.
 const LRUCACHE_SIZE = 10480
+
 type SyncMap struct {
-	Map   *lru.Cache[int, []byte] // Example data map
+	Map *lru.Cache[int, []byte] // Example data map
 }
 
 func NewSyncMap() *SyncMap {
@@ -45,7 +45,6 @@ func (s *SyncMap) String() string {
 	return res
 }
 
-
 func (s *SyncMap) Load(key int) ([]byte, bool) {
 	return s.Map.Get(key)
 }
@@ -68,8 +67,8 @@ func castBytesToBufSlice(buf []byte) (mem.BufferSlice, error) {
 }
 
 func init() {
-	log.Println("[INFO] Loading xchacha20-poly1305")
 	encoding.RegisterCodecV2(StreamDecryptCodec{})
+	slog.Debug("[cryptocodec] application-layer cryptography registered xchacha20-poly1305 gRPC codec")
 }
 
 type StreamDecryptCodec struct {
@@ -228,9 +227,9 @@ func (csvc *CryptoSvc) Encrypt(in_arr []byte) []byte {
 }
 
 type GoidTrace struct {
-	Id int
+	Id       int
 	ParentId int
-	Others []int
+	Others   []int
 }
 
 func goAllIds() (GoidTrace, error) {
@@ -248,9 +247,9 @@ func goAllIds() (GoidTrace, error) {
 		}
 	}
 	res := GoidTrace{
-		Id: ids[0],
+		Id:       ids[0],
 		ParentId: ids[1],
-		Others: ids[2:],
+		Others:   ids[2:],
 	}
 	return res, nil
 }