feat: replace config file with command args for network discovery (#33)

Author: leoparente

network-discovery/Makefile

@@ -8,7 +8,7 @@ COMMIT_SHA := $(shell git rev-parse --short HEAD)
 
 .PHONY: build
 build:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=$(GOARCH) GOARM=$(GOARM) go build -mod=mod -o ${BUILD_DIR}/network-discovery cmd/main.go
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) go build -mod=mod -o ${BUILD_DIR}/network-discovery cmd/main.go
 
 .PHONY: lint
 lint:

network-discovery/README.md

@@ -1,51 +1,58 @@
 # network-discovery
 Orb network discovery backend
 
-### Config RFC
-```yaml
-diode:
-  config:
-    target: grpc://localhost:8080/diode
-    api_key: ${DIODE_API_KEY}
-network_discovery:
-  config:
-    host: 0.0.0.0
-    port: 8072
-    log_level: info
-    log_format: json
+### Usage
+```sh
+Usage of network-discovery:
+  -diode-api-key string
+    	diode api key (REQUIRED). Environment variables can be used by wrapping them in ${} (e.g. ${MY_API_KEY})
+  -diode-target string
+    	diode target (REQUIRED)
+  -help
+    	show this help
+  -host string
+    	server host (default "0.0.0.0")
+  -log-format string
+    	log format (default "TEXT")
+  -log-level string
+    	log level (default "INFO")
+  -port int
+    	server port (default 8073)
 ```
 
 ### Policy RFC
 ```yaml
-network_discovery:
-  policies:
-    network_1:
-      config:
-        schedule: "* * * * *" #Cron expression
-        timeout: 5 #default 2 minutes
-      scope:
-        targets: [192.168.1.0/24]
-    discover_once: # will run only once
-      scope:
-         targets: 
-          - 192.168.0.34/24
-          - google.com
+policies:
+  network_1:
+    config:
+      schedule: "* * * * *" #Cron expression
+      timeout: 5 #default 2 minutes
+    scope:
+      targets: [192.168.1.0/24]
+  discover_once: # will run only once
+    scope:
+       targets: 
+        - 192.168.0.34/24
+        - google.com
 ```
 ## Run device-discovery
 device-discovery can be run by installing it with pip
 ```sh
 git clone https://github.com/netboxlabs/orb-discovery.git
 cd network-discovery/
 make bin
-build/network-discovery -c config.yaml
+build/network-discovery --diode-target grpc://192.168.31.114:8080/diode  --diode-api-key '${DIODE_API_KEY}'
 ```
 
 ## Docker Image
 device-discovery can be build and run using docker:
 ```sh
 cd network-discovery/
 docker build --no-cache -t network-discovery:develop -f docker/Dockerfile .
-docker run -v /local/orb:/usr/local/orb/ --net=host device-discovery:develop network-discovery -c /usr/local/orb/config.yaml
+docker run --net=host -e DIODE_API_KEY={YOUR_API_KEY} \
+ network-discovery:develop network-discovery \
+ --diode-target grpc://192.168.31.114:8080/diode \
+ --diode-api-key '${DIODE_API_KEY}'
 ```
 
 ### Routes (v1)

network-discovery/cmd/main.go

@@ -6,11 +6,10 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
-	"github.com/a8m/envsubst"
 	"github.com/netboxlabs/diode-sdk-go/diode"
-	"gopkg.in/yaml.v3"
 
 	"github.com/netboxlabs/orb-discovery/network-discovery/config"
 	"github.com/netboxlabs/orb-discovery/network-discovery/policy"
@@ -21,59 +20,60 @@ import (
 // AppName is the application name
 const AppName = "network-discovery"
 
+func resolveEnv(value string) string {
+	// Check if the value starts with ${ and ends with }
+	if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") {
+		// Extract the environment variable name
+		envVar := value[2 : len(value)-1]
+		// Get the value of the environment variable
+		envValue := os.Getenv(envVar)
+		if envValue != "" {
+			return envValue
+		}
+		fmt.Printf("error: environment variable %s is not set\n", envVar)
+		os.Exit(1)
+	}
+	// Return the original value if no substitution occurs
+	return value
+}
+
 func main() {
-	configPath := flag.String("config", "", "path to the configuration file (required)")
+	host := flag.String("host", "0.0.0.0", "server host")
+	port := flag.Int("port", 8073, "server port")
+	diodeTarget := flag.String("diode-target", "", "diode target (REQUIRED)")
+	diodeAPIKey := flag.String("diode-api-key", "", "diode api key (REQUIRED)."+
+		" Environment variables can be used by wrapping them in ${} (e.g. ${MY_API_KEY})")
+	logLevel := flag.String("log-level", "INFO", "log level")
+	logFormat := flag.String("log-format", "TEXT", "log format")
+	help := flag.Bool("help", false, "show this help")
 
 	flag.Parse()
 
-	if *configPath == "" {
+	if *help || *diodeTarget == "" || *diodeAPIKey == "" {
 		fmt.Fprintf(os.Stderr, "Usage of network-discovery:\n")
 		flag.PrintDefaults()
-		os.Exit(1)
-
-	}
-	if _, err := os.Stat(*configPath); os.IsNotExist(err) {
-		fmt.Printf("configuration file '%s' does not exist", *configPath)
-		os.Exit(1)
-	}
-	fileData, err := envsubst.ReadFile(*configPath)
-	if err != nil {
-		fmt.Printf("error reading configuration file: %v", err)
-		os.Exit(1)
-	}
-
-	cfg := config.Config{
-		Network: config.Network{
-			Config: config.StartupConfig{
-				Host:      "0.0.0.0",
-				Port:      8073,
-				LogLevel:  "INFO",
-				LogFormat: "TEXT",
-			}},
-	}
-
-	if err = yaml.Unmarshal(fileData, &cfg); err != nil {
-		fmt.Printf("error parsing configuration file: %v\n", err)
+		if *help {
+			os.Exit(0)
+		}
 		os.Exit(1)
 	}
 
 	client, err := diode.NewClient(
-		cfg.Diode.Config.Target,
+		*diodeTarget,
 		AppName,
 		version.GetBuildVersion(),
-		diode.WithAPIKey(cfg.Diode.Config.APIKey),
+		diode.WithAPIKey(resolveEnv(*diodeAPIKey)),
 	)
 	if err != nil {
 		fmt.Printf("error creating diode client: %v\n", err)
 		os.Exit(1)
 	}
 
 	ctx := context.Background()
-	logger := config.NewLogger(cfg.Network.Config.LogLevel, cfg.Network.Config.LogFormat)
+	logger := config.NewLogger(*logLevel, *logFormat)
 
 	policyManager := policy.NewManager(ctx, logger, client)
-	server := server.Server{}
-	server.Configure(logger, policyManager, version.GetBuildVersion(), cfg.Network.Config)
+	server := server.NewServer(*host, *port, logger, policyManager, version.GetBuildVersion())
 
 	// handle signals
 	done := make(chan bool, 1)

network-discovery/config/config.go

@@ -27,33 +27,7 @@ type Policy struct {
 	Scope  Scope        `yaml:"scope"`
 }
 
-// StartupConfig represents the configuration of the network-discovery service
-type StartupConfig struct {
-	Host      string `yaml:"host"`
-	Port      int32  `yaml:"port"`
-	LogLevel  string `yaml:"log_level"`
-	LogFormat string `yaml:"log_format"`
-}
-
-// Network represents the network-discovery configuration
-type Network struct {
-	Config   StartupConfig     `yaml:"config"`
+// Policies represents a collection of network-discovery policies
+type Policies struct {
 	Policies map[string]Policy `mapstructure:"policies"`
 }
-
-// DiodeConfig represents the configuration of diode service
-type DiodeConfig struct {
-	Target string `yaml:"target"`
-	APIKey string `yaml:"api_key"`
-}
-
-// Diode represents the root configuration of diode service
-type Diode struct {
-	Config DiodeConfig `yaml:"config"`
-}
-
-// Config represents the root configuration of the network-discovery service
-type Config struct {
-	Diode   Diode   `yaml:"diode"`
-	Network Network `yaml:"network_discovery"`
-}

network-discovery/go.mod

@@ -4,7 +4,6 @@ go 1.23.3
 
 require (
 	github.com/Ullaakut/nmap/v3 v3.0.4
-	github.com/a8m/envsubst v1.4.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-co-op/gocron/v2 v2.12.4
 	github.com/netboxlabs/diode-sdk-go v0.2.0

network-discovery/go.sum

@@ -1,7 +1,5 @@
 github.com/Ullaakut/nmap/v3 v3.0.4 h1:ZwRM4gbE0gtoXipC7uTHs2wvf3r4S8d9mtqfTabXE34=
 github.com/Ullaakut/nmap/v3 v3.0.4/go.mod h1:dd5K68P7LHc5nKrFwQx6EdTt61O9UN5x3zn1R4SLcco=
-github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
-github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=

network-discovery/policy/manager.go

@@ -32,16 +32,16 @@ func NewManager(ctx context.Context, logger *slog.Logger, client diode.Client) *
 
 // ParsePolicies parses the policies from the request
 func (m *Manager) ParsePolicies(data []byte) (map[string]config.Policy, error) {
-	var payload config.Config
+	var payload config.Policies
 	if err := yaml.Unmarshal(data, &payload); err != nil {
 		return nil, err
 	}
 
-	if payload.Network.Policies == nil {
+	if len(payload.Policies) == 0 {
 		return nil, errors.New("no policies found in the request")
 	}
 
-	return payload.Network.Policies, nil
+	return payload.Policies, nil
 }
 
 // HasPolicy checks if the policy exists

network-discovery/policy/manager_test.go

@@ -38,15 +38,14 @@ func TestManagerParsePolicies(t *testing.T) {
 
 	t.Run("Valid Policies", func(t *testing.T) {
 		yamlData := []byte(`
-        network_discovery:
-          policies:
-            policy1:
-              config:
-                defaults:
-                  site: New York NY
-              scope:
-                targets:
-                  - 192.168.1.1/24
+        policies:
+          policy1:
+            config:
+              defaults:
+                site: New York NY
+            scope:
+              targets:
+                - 192.168.1.1/24
        `)
 
 		policies, err := manager.ParsePolicies(yamlData)
@@ -67,19 +66,18 @@ func TestManagerPolicyLifecycle(t *testing.T) {
 	logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false}))
 	manager := policy.NewManager(context.Background(), logger, nil)
 	yamlData := []byte(`
-        network_discovery:
-          policies:
-            policy1:
-              scope:
-                targets:
-                  - 192.168.1.1/24
-            policy2:
-              scope:
-                targets:
-                  - 192.168.2.1/24
-            policy3:
-              scope:
-                targets: []
+        policies:
+          policy1:
+            scope:
+              targets:
+                - 192.168.1.1/24
+          policy2:
+            scope:
+              targets:
+                - 192.168.2.1/24
+          policy3:
+            scope:
+              targets: []
        `)
 
 	policies, err := manager.ParsePolicies(yamlData)

network-discovery/policy/runner.go

@@ -81,7 +81,7 @@ func (r *Runner) run() {
 		r.logger.Error("error creating scanner", slog.Any("error", err), slog.Any("policy", r.ctx.Value(policyKey)))
 		return
 	}
-
+	r.logger.Info("running scanner", slog.Any("targets", r.scope.Targets), slog.Any("policy", r.ctx.Value(policyKey)))
 	result, warnings, err := scanner.Run()
 	if len(*warnings) > 0 {
 		r.logger.Warn("run finished with warnings", slog.String("warnings", fmt.Sprintf("%v", *warnings)))

network-discovery/server/server.go

@@ -31,30 +31,37 @@ type Server struct {
 	manager *policy.Manager
 	stat    config.Status
 	logger  *slog.Logger
-	config  config.StartupConfig
+	host    string
+	port    int
 }
 
 func init() {
 	gin.SetMode(gin.ReleaseMode)
 }
 
-// Configure configures the network-discovery server
-func (s *Server) Configure(logger *slog.Logger, manager *policy.Manager, version string, config config.StartupConfig) {
-	s.stat.Version = version
-	s.stat.StartTime = time.Now()
-	s.manager = manager
-	s.logger = logger
-	s.config = config
-
-	s.router = gin.New()
+// NewServer returns a new network-discovery server
+func NewServer(host string, port int, logger *slog.Logger, manager *policy.Manager, version string) *Server {
+	server := &Server{
+		router:  gin.New(),
+		manager: manager,
+		stat: config.Status{
+			Version:   version,
+			StartTime: time.Now(),
+		},
+		logger: logger,
+		host:   host,
+		port:   port,
+	}
 
-	v1 := s.router.Group("/api/v1")
+	v1 := server.router.Group("/api/v1")
 	{
-		v1.GET("/status", s.getStatus)
-		v1.GET("/capabilities", s.getCapabilities)
-		v1.POST("/policies", s.createPolicy)
-		v1.DELETE("/policies/:policy", s.deletePolicy)
+		v1.GET("/status", server.getStatus)
+		v1.GET("/capabilities", server.getCapabilities)
+		v1.POST("/policies", server.createPolicy)
+		v1.DELETE("/policies/:policy", server.deletePolicy)
 	}
+
+	return server
 }
 
 // Router returns the router
@@ -65,7 +72,7 @@ func (s *Server) Router() *gin.Engine {
 // Start starts the network-discovery server
 func (s *Server) Start() {
 	go func() {
-		serv := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port)
+		serv := fmt.Sprintf("%s:%d", s.host, s.port)
 		s.logger.Info("starting network-discovery server at: " + serv)
 		if err := s.router.Run(serv); err != nil {
 			s.logger.Error("shutting down the server", "error", err)