Implement repository detection feature in search-servers command (#30)

* Implement repository detection feature in search-servers command

- Added support for automatic detection of npm and PyPI packages in the search-servers command.
- Enhanced command description and usage examples to reflect new functionality.
- Introduced configuration option `check_server_repo` to enable/disable repository detection.
- Updated SearchServers function to accept a limit parameter and a guesser for repository info.
- Added comprehensive documentation for the repository detection feature, including configuration and usage examples.

* Refactor repository guessing to support only GitHub URLs

- Updated GuessRepositoryType to handle only GitHub URLs, removing PyPI support.
- Simplified package name extraction for npm packages in the format @author/repo.
- Enhanced tests to validate new GitHub URL handling and npm package detection.
- Removed unused PyPI-related code and tests to streamline functionality.

* Refactor test setup to use temporary database files for compatibility

- Updated test setup functions in guesser_test.go and search_test.go to create temporary database files instead of using in-memory databases, ensuring compatibility across platforms.

* Implement batch processing for repository guessing in mcpproxy

- Introduced batch processing for repository guessing in the GuessRepositoryTypesBatch method, allowing concurrent checks of multiple GitHub URLs.
- Enhanced performance by implementing connection pooling and limiting concurrent requests to improve response times.
- Updated SearchServers to utilize batch processing for repository guessing, applying it only to filtered results.
- Refactored related parsing functions to support new batch processing without impacting existing functionality.
- Added comprehensive tests to validate batch processing behavior and performance improvements.

* Enhance logging configuration and command handling in mcpproxy

- Updated default logging settings to disable file logging and enable console output.
- Modified command logger setup to use appropriate log levels based on command type.
- Improved logging in search-servers command to provide detailed information on server searches and registry loading.
- Adjusted autostart configurations to include logging options for better traceability.

* Enhance logging in GuessRepositoryTypesBatch for npm package results

- Added detailed logging for npm package detection results, including package name, version, and installation command.
- Improved error handling logging for failed repository type guesses, ensuring clarity on the URL and error details.
- Streamlined logging for cases with no npm package information.

* Refactor logging and error handling in mcpproxy and guesser

- Introduced a default log level constant in mcpproxy for improved logging consistency.
- Updated listAllRegistries function to remove error return, simplifying its signature.
- Enhanced guessRepositoryTypeBatch to eliminate error handling, returning only results.
- Streamlined server iteration in search.go for better readability.

* Enhance search_servers documentation and update server entry structure

- Updated documentation to include repository information and separate MCP endpoints from source code repositories.
- Added `source_code_url` field to `ServerEntry` struct for better clarity on source code links.
- Modified tests to validate the new `source_code_url` field and ensure correct parsing of server entries.

* Enhance OAuth configuration handling and add base URL parsing

- Updated CreateOAuthConfig to construct the OAuth server metadata URL from the server URL.
- Introduced parseBaseURL function to extract the base URL from a full URL, handling cases without a scheme.
- Improved logging for OAuth server metadata URL setting and error handling for URL parsing failures.

Author: Dumbris

cmd/mcpproxy/main.go

@@ -13,6 +13,7 @@ import (
 	"go.uber.org/zap"
 
 	"mcpproxy-go/internal/config"
+	"mcpproxy-go/internal/experiments"
 	"mcpproxy-go/internal/logs"
 	"mcpproxy-go/internal/registries"
 	"mcpproxy-go/internal/server"
@@ -39,6 +40,10 @@ var (
 	version = "v0.1.0" // This will be injected by -ldflags during build
 )
 
+const (
+	defaultLogLevel = "info"
+)
+
 // TrayInterface defines the interface for system tray functionality
 type TrayInterface interface {
 	Run(ctx context.Context) error
@@ -59,8 +64,8 @@ func main() {
 	// Add global flags
 	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "Configuration file path")
 	rootCmd.PersistentFlags().StringVarP(&dataDir, "data-dir", "d", "", "Data directory path (default: ~/.mcpproxy)")
-	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
-	rootCmd.PersistentFlags().BoolVar(&logToFile, "log-to-file", true, "Enable logging to file in standard OS location")
+	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "", "Log level (debug, info, warn, error) - defaults: server=info, other commands=warn")
+	rootCmd.PersistentFlags().BoolVar(&logToFile, "log-to-file", false, "Enable logging to file in standard OS location (default: console only)")
 	rootCmd.PersistentFlags().StringVar(&logDir, "log-dir", "", "Custom log directory path (overrides standard OS location)")
 
 	// Add server command
@@ -101,38 +106,81 @@ func main() {
 func createSearchServersCommand() *cobra.Command {
 	var registryFlag, searchFlag, tagFlag string
 	var listRegistries bool
+	var limitFlag int
 
 	cmd := &cobra.Command{
 		Use:   "search-servers",
-		Short: "Search MCP registries for available servers",
+		Short: "Search MCP registries for available servers with repository detection",
 		Long: `Search known MCP registries for available servers that can be added as upstreams.
-This tool queries embedded registries to discover MCP servers and returns results
-that can be directly used with the 'upstream_servers add' command.
+This tool queries embedded registries to discover MCP servers and includes automatic
+npm/PyPI package detection to enhance results with install commands.
+Results can be directly used with the 'upstream_servers add' command.
 
 Examples:
   # List all known registries
   mcpproxy search-servers --list-registries
 
-  # Search for weather-related servers in the Smithery registry
-  mcpproxy search-servers --registry smithery --search weather
+  # Search for weather-related servers in the Smithery registry (limit 10 results)
+  mcpproxy search-servers --registry smithery --search weather --limit 10
 
   # Search for servers tagged as "finance" in the Pulse registry
   mcpproxy search-servers --registry pulse --tag finance`,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			// Setup logger for search command (non-server command = WARN level by default)
+			cmdLogLevel, _ := cmd.Flags().GetString("log-level")
+			cmdLogToFile, _ := cmd.Flags().GetBool("log-to-file")
+			cmdLogDir, _ := cmd.Flags().GetString("log-dir")
+
+			logger, err := logs.SetupCommandLogger(false, cmdLogLevel, cmdLogToFile, cmdLogDir)
+			if err != nil {
+				return fmt.Errorf("failed to setup logger: %w", err)
+			}
+			defer func() {
+				_ = logger.Sync()
+			}()
+
 			if listRegistries {
-				return listAllRegistries()
+				listAllRegistries(logger)
+				return nil
 			}
 
 			if registryFlag == "" {
 				return fmt.Errorf("--registry is required (use --list-registries to see available options)")
 			}
 
 			ctx := context.Background()
-			servers, err := registries.SearchServers(ctx, registryFlag, tagFlag, searchFlag)
+
+			// Create config to check if repository guessing is enabled
+			cfg, err := config.LoadFromFile("")
 			if err != nil {
+				// Use default config if loading fails
+				cfg = config.DefaultConfig()
+			}
+
+			// Initialize registries from config
+			registries.SetRegistriesFromConfig(cfg)
+
+			// Create experiments guesser if repository checking is enabled
+			var guesser *experiments.Guesser
+			if cfg.CheckServerRepo {
+				// Use the proper logger instead of no-op
+				guesser = experiments.NewGuesser(nil, logger)
+			}
+
+			logger.Info("Searching servers",
+				zap.String("registry", registryFlag),
+				zap.String("search", searchFlag),
+				zap.String("tag", tagFlag),
+				zap.Int("limit", limitFlag))
+
+			servers, err := registries.SearchServers(ctx, registryFlag, tagFlag, searchFlag, limitFlag, guesser)
+			if err != nil {
+				logger.Error("Search failed", zap.Error(err))
 				return fmt.Errorf("search failed: %w", err)
 			}
 
+			logger.Info("Search completed", zap.Int("results_count", len(servers)))
+
 			// Print results as JSON
 			output, err := json.MarshalIndent(servers, "", "  ")
 			if err != nil {
@@ -147,14 +195,29 @@ Examples:
 	cmd.Flags().StringVarP(&registryFlag, "registry", "r", "", "Registry ID or name to search (exact match)")
 	cmd.Flags().StringVarP(&searchFlag, "search", "s", "", "Search term for server name/description")
 	cmd.Flags().StringVarP(&tagFlag, "tag", "t", "", "Filter servers by tag/category")
+	cmd.Flags().IntVarP(&limitFlag, "limit", "l", 10, "Maximum number of results to return (default: 10, max: 50)")
 	cmd.Flags().BoolVar(&listRegistries, "list-registries", false, "List all known registries")
 
 	return cmd
 }
 
-func listAllRegistries() error {
+func listAllRegistries(logger *zap.Logger) {
+	// Load config and initialize registries
+	cfg, err := config.LoadFromFile("")
+	if err != nil {
+		// Use default config if loading fails
+		cfg = config.DefaultConfig()
+	}
+
+	logger.Info("Loading registries configuration")
+
+	// Initialize registries from config
+	registries.SetRegistriesFromConfig(cfg)
+
 	registryList := registries.ListRegistries()
 
+	logger.Info("Found registries", zap.Int("count", len(registryList)))
+
 	// Format as a simple table for CLI display
 	fmt.Printf("%-20s %-30s %s\n", "ID", "NAME", "DESCRIPTION")
 	fmt.Printf("%-20s %-30s %s\n", "==", "====", "===========")
@@ -165,7 +228,6 @@ func listAllRegistries() error {
 	}
 
 	fmt.Printf("\nUse --registry <ID> to search a specific registry\n")
-	return nil
 }
 
 func runServer(cmd *cobra.Command, _ []string) error {
@@ -190,8 +252,14 @@ func runServer(cmd *cobra.Command, _ []string) error {
 
 	// Override logging settings from command line
 	if cfg.Logging == nil {
+		// Use command-specific default level (INFO for server command)
+		defaultLevel := cmdLogLevel
+		if defaultLevel == "" {
+			defaultLevel = defaultLogLevel // Server command defaults to INFO
+		}
+
 		cfg.Logging = &config.LogConfig{
-			Level:         cmdLogLevel,
+			Level:         defaultLevel,
 			EnableFile:    cmdLogToFile,
 			EnableConsole: true,
 			Filename:      "main.log",
@@ -203,7 +271,11 @@ func runServer(cmd *cobra.Command, _ []string) error {
 		}
 	} else {
 		// Override specific fields from command line
-		cfg.Logging.Level = cmdLogLevel
+		if cmdLogLevel != "" {
+			cfg.Logging.Level = cmdLogLevel
+		} else if cfg.Logging.Level == "" {
+			cfg.Logging.Level = defaultLogLevel // Server command defaults to INFO
+		}
 		cfg.Logging.EnableFile = cmdLogToFile
 		if cfg.Logging.Filename == "" || cfg.Logging.Filename == "mcpproxy.log" {
 			cfg.Logging.Filename = "main.log"