bugfixes and default config optimization

Author: ndyakov

async_handoff_integration_test.go

@@ -17,12 +17,12 @@ type mockNetConn struct {
 }
 
 func (m *mockNetConn) Read(b []byte) (n int, err error)   { return 0, nil }
-func (m *mockNetConn) Write(b []byte) (n int, err error) { return len(b), nil }
-func (m *mockNetConn) Close() error                      { return nil }
-func (m *mockNetConn) LocalAddr() net.Addr               { return &mockAddr{m.addr} }
-func (m *mockNetConn) RemoteAddr() net.Addr              { return &mockAddr{m.addr} }
-func (m *mockNetConn) SetDeadline(t time.Time) error     { return nil }
-func (m *mockNetConn) SetReadDeadline(t time.Time) error { return nil }
+func (m *mockNetConn) Write(b []byte) (n int, err error)  { return len(b), nil }
+func (m *mockNetConn) Close() error                       { return nil }
+func (m *mockNetConn) LocalAddr() net.Addr                { return &mockAddr{m.addr} }
+func (m *mockNetConn) RemoteAddr() net.Addr               { return &mockAddr{m.addr} }
+func (m *mockNetConn) SetDeadline(t time.Time) error      { return nil }
+func (m *mockNetConn) SetReadDeadline(t time.Time) error  { return nil }
 func (m *mockNetConn) SetWriteDeadline(t time.Time) error { return nil }
 
 type mockAddr struct {
@@ -152,8 +152,8 @@ func TestEventDrivenHandoffIntegration(t *testing.T) {
 				return &mockNetConn{addr: "original:6379"}, nil
 			},
 
-			PoolSize:       int32(10),
-			PoolTimeout:    time.Second,
+			PoolSize:    int32(10),
+			PoolTimeout: time.Second,
 		})
 		defer testPool.Close()
 
@@ -175,7 +175,7 @@ func TestEventDrivenHandoffIntegration(t *testing.T) {
 				// Get connection
 				conn, err := testPool.Get(ctx)
 				if err != nil {
-					t.Errorf("Failed to get connection %d: %v", id, err)
+					t.Errorf("Failed to get conn[%d]: %v", id, err)
 					return
 				}
 
@@ -224,8 +224,8 @@ func TestEventDrivenHandoffIntegration(t *testing.T) {
 				return &mockNetConn{addr: "original:6379"}, nil
 			},
 
-			PoolSize:       int32(3),
-			PoolTimeout:    time.Second,
+			PoolSize:    int32(3),
+			PoolTimeout: time.Second,
 		})
 		defer testPool.Close()
 
@@ -287,8 +287,8 @@ func TestEventDrivenHandoffIntegration(t *testing.T) {
 				return &mockNetConn{addr: "original:6379"}, nil
 			},
 
-			PoolSize:       int32(2),
-			PoolTimeout:    time.Second,
+			PoolSize:    int32(2),
+			PoolTimeout: time.Second,
 		})
 		defer testPool.Close()
 

hitless/README.md

@@ -39,27 +39,31 @@ Config: &hitless.Config{
     PostHandoffRelaxedDuration: 20 * time.Second, // Keep relaxed timeout after handoff
     LogLevel:                   logging.LogLevelWarn, // LogLevelError, LogLevelWarn, LogLevelInfo, LogLevelDebug
     MaxWorkers:                 15, // Concurrent handoff workers
-    HandoffQueueSize:           50, // Handoff request queue size
+    HandoffQueueSize:           300, // Handoff request queue size
 }
 ```
 
 ### Worker Scaling
-- **Auto-calculated**: `min(10, PoolSize/3)` - scales with pool size, capped at 10
-- **Explicit values**: `max(10, set_value)` - enforces minimum 10 workers
+- **Auto-calculated**: `min(PoolSize/2, max(10, PoolSize/3))` - balanced scaling approach
+- **Explicit values**: `max(PoolSize/2, set_value)` - enforces minimum PoolSize/2 workers
 - **On-demand**: Workers created when needed, cleaned up when idle
 
 ### Queue Sizing
-- **Auto-calculated**: `max(8 × MaxWorkers, max(50, PoolSize/2))` - hybrid scaling
-  - Worker-based: 8 handoffs per worker for burst processing
-  - Pool-based: Scales with pool size (minimum 50, up to PoolSize/2)
+- **Auto-calculated**: `max(20 × MaxWorkers, PoolSize)` - hybrid scaling
+  - Worker-based: 20 handoffs per worker for burst processing
+  - Pool-based: Scales directly with pool size
   - Takes the larger of the two for optimal performance
-- **Explicit values**: `max(50, set_value)` - enforces minimum 50 when set
-- **Always capped**: Queue size never exceeds `2 × PoolSize` for memory efficiency
+- **Explicit values**: `max(200, set_value)` - enforces minimum 200 when set
+- **Capping**: Queue size capped by `MaxActiveConns+1` (if set) or `5 × PoolSize` for memory efficiency
 
-**Examples:**
-- Pool 10: Queue 50 (max(8×3, max(50, 5)) = max(24, 50) = 50)
-- Pool 100: Queue 80 (max(8×10, max(50, 50)) = max(80, 50) = 80)
-- Pool 200: Queue 100 (max(8×10, max(50, 100)) = max(80, 100) = 100)
+**Examples (without MaxActiveConns):**
+- Pool 10: Workers 5, Queue 100 (max(20×5, 10) = 100, capped at 5×10 = 50)
+- Pool 100: Workers 33, Queue 660 (max(20×33, 100) = 660, capped at 5×100 = 500)
+- Pool 200: Workers 66, Queue 1320 (max(20×66, 200) = 1320, capped at 5×200 = 1000)
+
+**Examples (with MaxActiveConns=150):**
+- Pool 100: Workers 33, Queue 151 (max(20×33, 100) = 660, capped at MaxActiveConns+1 = 151)
+- Pool 200: Workers 66, Queue 151 (max(20×66, 200) = 1320, capped at MaxActiveConns+1 = 151)
 
 ## Notification Hooks
 
@@ -94,7 +98,7 @@ type CustomHook struct{}
 func (h *CustomHook) PreHook(ctx context.Context, notificationCtx push.NotificationHandlerContext, notificationType string, notification []interface{}) ([]interface{}, bool) {
     // Log notification with connection details
     if conn, ok := notificationCtx.Conn.(*pool.Conn); ok {
-        log.Printf("Processing %s on connection %d", notificationType, conn.GetID())
+        log.Printf("Processing %s on conn[%d]", notificationType, conn.GetID())
     }
     return notification, true // Continue processing
 }

hitless/config.go

@@ -103,30 +103,25 @@ type Config struct {
 
 	// MaxWorkers is the maximum number of worker goroutines for processing handoff requests.
 	// Workers are created on-demand and automatically cleaned up when idle.
-	// If zero, defaults to min(10, PoolSize/3) to handle bursts effectively.
-	// If explicitly set, enforces minimum of 10 workers.
+	// If zero, defaults to min(10, PoolSize/2) to handle bursts effectively.
+	// If explicitly set, enforces minimum of PoolSize/2
 	//
-	// Default: min(10, PoolSize/3), Minimum when set: 10
+	// Default: min(PoolSize/2, max(10, PoolSize/3)), Minimum when set: PoolSize/2
 	MaxWorkers int
 
 	// HandoffQueueSize is the size of the buffered channel used to queue handoff requests.
 	// If the queue is full, new handoff requests will be rejected.
 	// Scales with both worker count and pool size for better burst handling.
 	//
-	// Default: max(8x workers, max(50, PoolSize/2)), capped by 2x pool size
-	// When set: min 50, capped by 2x pool size
+	// Default: max(20×MaxWorkers, PoolSize), capped by MaxActiveConns+1 (if set) or 5×PoolSize
+	// When set: minimum 200, capped by MaxActiveConns+1 (if set) or 5×PoolSize
 	HandoffQueueSize int
 
 	// PostHandoffRelaxedDuration is how long to keep relaxed timeouts on the new connection
 	// after a handoff completes. This provides additional resilience during cluster transitions.
 	// Default: 2 * RelaxedTimeout
 	PostHandoffRelaxedDuration time.Duration
 
-	// ScaleDownDelay is the delay before checking if workers should be scaled down.
-	// This prevents expensive checks on every handoff completion and avoids rapid scaling cycles.
-	// Default: 2 seconds
-	ScaleDownDelay time.Duration
-
 	// LogLevel controls the verbosity of hitless upgrade logging.
 	// LogLevelError (0) = errors only, LogLevelWarn (1) = warnings, LogLevelInfo (2) = info, LogLevelDebug (3) = debug
 	// Default: LogLevelWarn (warnings)
@@ -152,7 +147,6 @@ func DefaultConfig() *Config {
 		MaxWorkers:                 0, // Auto-calculated based on pool size
 		HandoffQueueSize:           0, // Auto-calculated based on max workers
 		PostHandoffRelaxedDuration: 0, // Auto-calculated based on relaxed timeout
-		ScaleDownDelay:             2 * time.Second,
 		LogLevel:                   LogLevelWarn,
 
 		// Connection Handoff Configuration
@@ -212,6 +206,13 @@ func (c *Config) ApplyDefaults() *Config {
 // using the provided pool size to calculate worker defaults.
 // This ensures that partially configured structs get sensible defaults for missing fields.
 func (c *Config) ApplyDefaultsWithPoolSize(poolSize int) *Config {
+	return c.ApplyDefaultsWithPoolConfig(poolSize, 0)
+}
+
+// ApplyDefaultsWithPoolConfig applies default values to any zero-value fields in the configuration,
+// using the provided pool size and max active connections to calculate worker and queue defaults.
+// This ensures that partially configured structs get sensible defaults for missing fields.
+func (c *Config) ApplyDefaultsWithPoolConfig(poolSize int, maxActiveConns int) *Config {
 	if c == nil {
 		return DefaultConfig().ApplyDefaultsWithPoolSize(poolSize)
 	}
@@ -251,19 +252,25 @@ func (c *Config) ApplyDefaultsWithPoolSize(poolSize int) *Config {
 	// Apply worker defaults based on pool size
 	result.applyWorkerDefaults(poolSize)
 
-	// Apply queue size defaults with hybrid scaling approach
+	// Apply queue size defaults with new scaling approach
 	if c.HandoffQueueSize <= 0 {
-		// Default: max(8x workers, max(50, PoolSize/2)), capped by 2x pool size
-		workerBasedSize := result.MaxWorkers * 8
-		poolBasedSize := util.Max(50, poolSize/2)
+		// Default: max(20x workers, PoolSize), capped by maxActiveConns or 5x pool size
+		workerBasedSize := result.MaxWorkers * 20
+		poolBasedSize := poolSize
 		result.HandoffQueueSize = util.Max(workerBasedSize, poolBasedSize)
 	} else {
-		// When explicitly set: enforce minimum of 50
-		result.HandoffQueueSize = util.Max(50, c.HandoffQueueSize)
+		// When explicitly set: enforce minimum of 200
+		result.HandoffQueueSize = util.Max(200, c.HandoffQueueSize)
 	}
 
-	// Always cap queue size by 2x pool size - balances burst capacity with memory efficiency
-	result.HandoffQueueSize = util.Min(result.HandoffQueueSize, poolSize*2)
+	// Cap queue size: use maxActiveConns+1 if set, otherwise 5x pool size
+	var queueCap int
+	if maxActiveConns > 0 {
+		queueCap = maxActiveConns + 1
+	} else {
+		queueCap = poolSize * 5
+	}
+	result.HandoffQueueSize = util.Min(result.HandoffQueueSize, queueCap)
 
 	// Ensure minimum queue size of 2 (fallback for very small pools)
 	if result.HandoffQueueSize < 2 {
@@ -276,12 +283,6 @@ func (c *Config) ApplyDefaultsWithPoolSize(poolSize int) *Config {
 		result.PostHandoffRelaxedDuration = c.PostHandoffRelaxedDuration
 	}
 
-	if c.ScaleDownDelay <= 0 {
-		result.ScaleDownDelay = defaults.ScaleDownDelay
-	} else {
-		result.ScaleDownDelay = c.ScaleDownDelay
-	}
-
 	// LogLevel: 0 is a valid value (errors only), so we need to check if it was explicitly set
 	// We'll use the provided value as-is, since 0 is valid
 	result.LogLevel = c.LogLevel
@@ -314,7 +315,6 @@ func (c *Config) Clone() *Config {
 		MaxWorkers:                 c.MaxWorkers,
 		HandoffQueueSize:           c.HandoffQueueSize,
 		PostHandoffRelaxedDuration: c.PostHandoffRelaxedDuration,
-		ScaleDownDelay:             c.ScaleDownDelay,
 		LogLevel:                   c.LogLevel,
 
 		// Configuration fields
@@ -330,11 +330,11 @@ func (c *Config) applyWorkerDefaults(poolSize int) {
 	}
 
 	if c.MaxWorkers == 0 {
-		// When not set: min(10, poolSize/3) - don't exceed 10 workers for small pools
-		c.MaxWorkers = util.Min(10, poolSize/3)
+		// When not set: min(poolSize/2, max(10, poolSize/3)) - balanced scaling approach
+		c.MaxWorkers = util.Min(poolSize/2, util.Max(10, poolSize/3))
 	} else {
-		// When explicitly set: max(10, set_value) - ensure at least 10 workers
-		c.MaxWorkers = util.Max(10, c.MaxWorkers)
+		// When explicitly set: max(poolSize/2, set_value) - ensure at least poolSize/2 workers
+		c.MaxWorkers = util.Max(poolSize/2, c.MaxWorkers)
 	}
 
 	// Ensure minimum of 1 worker (fallback for very small pools)

hitless/errors.go

@@ -43,3 +43,8 @@ var (
 	// ErrConnectionInvalidHandoffState is returned when a connection is in an invalid state for handoff
 	ErrConnectionInvalidHandoffState = errors.New("hitless: connection is in invalid state for handoff")
 )
+
+// general errors
+var (
+	ErrShutdown = errors.New("hitless: shutdown")
+)

hitless/example_hooks.go

@@ -38,7 +38,7 @@ func (mh *MetricsHook) PreHook(ctx context.Context, notificationCtx push.Notific
 
 	// Log connection information if available
 	if conn, ok := notificationCtx.Conn.(*pool.Conn); ok {
-		internal.Logger.Printf(ctx, "hitless: metrics hook processing %s notification on connection %d", notificationType, conn.GetID())
+		internal.Logger.Printf(ctx, "hitless: metrics hook processing %s notification on conn[%d]", notificationType, conn.GetID())
 	}
 
 	// Store start time in context for duration calculation
@@ -62,7 +62,7 @@ func (mh *MetricsHook) PostHook(ctx context.Context, notificationCtx push.Notifi
 
 		// Log error details with connection information
 		if conn, ok := notificationCtx.Conn.(*pool.Conn); ok {
-			internal.Logger.Printf(ctx, "hitless: metrics hook recorded error for %s notification on connection %d: %v", notificationType, conn.GetID(), result)
+			internal.Logger.Printf(ctx, "hitless: metrics hook recorded error for %s notification on conn[%d]: %v", notificationType, conn.GetID(), result)
 		}
 	}
 }