feat: add extensible observability via metrics, tracing, and logging

README.md

@@ -43,7 +43,7 @@ A flexible HTTP client with automatic retry logic using exponential backoff, bui
 - **Request Options**: Flexible request configuration with `WithBody()`, `WithJSON()`, `WithHeader()`, and `WithHeaders()`
 - **Jitter Support**: Optional random jitter to prevent thundering herd problem
 - **Retry-After Header**: Respects HTTP `Retry-After` header for rate limiting (RFC 2616)
-- **Observability Hooks**: Callback functions for logging, metrics, and custom retry logic
+- **Observability**: Built-in support for metrics collection, distributed tracing, and structured logging (zero dependencies, interface-driven)
 - **Flexible Configuration**: Use functional options to customize retry behavior
 - **Context Support**: Respects context cancellation and timeouts
 - **Custom Retry Logic**: Pluggable retry checker for custom retry conditions
@@ -205,6 +205,7 @@ For detailed documentation, please refer to:
 - **[Preset Configurations](docs/PRESETS.md)** - Pre-configured clients for common scenarios (realtime, background, rate-limited, microservice, webhook, critical, fast-fail, etc.)
 - **[Configuration Options](docs/CONFIGURATION.md)** - All available configuration options including retry behavior, HTTP client settings, custom TLS, and request options
 - **[Error Handling](docs/ERROR_HANDLING.md)** - Structured error handling with `RetryError` and response inspection
+- **[Observability](docs/OBSERVABILITY.md)** - Metrics collection, distributed tracing, and structured logging (OpenTelemetry, Prometheus, slog integration patterns)
 - **[Examples](docs/EXAMPLES.md)** - Detailed usage examples for various scenarios
 
 ### Key Topics
@@ -269,7 +270,8 @@ if err != nil {
 For complete, runnable examples, see:
 
 - [\_example/basic](_example/basic) - Basic usage with default settings
-- [\_example/advanced](_example/advanced) - Advanced configuration with observability and custom retry logic
+- [\_example/advanced](_example/advanced) - Advanced configuration with custom retry logic
+- [\_example/observability](_example/observability) - Metrics, tracing, and logging integration patterns (Prometheus, OpenTelemetry, slog)
 - [\_example/convenience_methods](_example/convenience_methods) - Using convenience HTTP methods (GET, POST, PUT, DELETE, HEAD, PATCH)
 - [\_example/request_options](_example/request_options) - Request options usage (WithBody, WithJSON, WithHeader, WithHeaders)
 - [\_example/large_file_upload](_example/large_file_upload) - ⚠️ **Important**: Correct way to upload large files (>10MB) with retry support

_example/observability/combined/main.go

@@ -0,0 +1,207 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"time"
+
+	retry "github.com/appleboy/go-httpretry"
+)
+
+// SimpleMetricsCollector demonstrates a basic metrics implementation
+type SimpleMetricsCollector struct {
+	mu               sync.Mutex
+	totalAttempts    int
+	totalRetries     int
+	successfulReqs   int
+	failedReqs       int
+	totalDuration    time.Duration
+	attemptDurations []time.Duration
+}
+
+func (m *SimpleMetricsCollector) RecordAttempt(method string, statusCode int, duration time.Duration, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.totalAttempts++
+	m.attemptDurations = append(m.attemptDurations, duration)
+	fmt.Printf("[METRIC] Attempt recorded: method=%s status=%d duration=%v err=%v\n",
+		method, statusCode, duration, err)
+}
+
+func (m *SimpleMetricsCollector) RecordRetry(method string, reason string, attemptNumber int) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.totalRetries++
+	fmt.Printf("[METRIC] Retry recorded: method=%s reason=%s attempt=%d\n",
+		method, reason, attemptNumber)
+}
+
+func (m *SimpleMetricsCollector) RecordRequestComplete(method string, statusCode int, totalDuration time.Duration, totalAttempts int, success bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.totalDuration += totalDuration
+	if success {
+		m.successfulReqs++
+	} else {
+		m.failedReqs++
+	}
+	fmt.Printf("[METRIC] Request complete: method=%s status=%d duration=%v attempts=%d success=%v\n",
+		method, statusCode, totalDuration, totalAttempts, success)
+}
+
+func (m *SimpleMetricsCollector) PrintSummary() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	fmt.Println("\n=== METRICS SUMMARY ===")
+	fmt.Printf("Total Attempts: %d\n", m.totalAttempts)
+	fmt.Printf("Total Retries: %d\n", m.totalRetries)
+	fmt.Printf("Successful Requests: %d\n", m.successfulReqs)
+	fmt.Printf("Failed Requests: %d\n", m.failedReqs)
+	fmt.Printf("Total Duration: %v\n", m.totalDuration)
+	if len(m.attemptDurations) > 0 {
+		var sum time.Duration
+		for _, d := range m.attemptDurations {
+			sum += d
+		}
+		avg := sum / time.Duration(len(m.attemptDurations))
+		fmt.Printf("Average Attempt Duration: %v\n", avg)
+	}
+}
+
+// SimpleTracer demonstrates a basic tracing implementation
+type SimpleTracer struct {
+	mu    sync.Mutex
+	spans []*SimpleSpan
+}
+
+type SimpleSpan struct {
+	mu          sync.Mutex
+	name        string
+	startTime   time.Time
+	endTime     time.Time
+	attributes  []retry.Attribute
+	status      string
+	description string
+	events      []SimpleEvent
+}
+
+type SimpleEvent struct {
+	name       string
+	timestamp  time.Time
+	attributes []retry.Attribute
+}
+
+func (t *SimpleTracer) StartSpan(ctx context.Context, operationName string, attrs ...retry.Attribute) (context.Context, retry.Span) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	span := &SimpleSpan{
+		name:       operationName,
+		startTime:  time.Now(),
+		attributes: attrs,
+	}
+	t.spans = append(t.spans, span)
+
+	fmt.Printf("[TRACE] Span started: name=%s\n", operationName)
+	return ctx, span
+}
+
+func (s *SimpleSpan) End() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.endTime = time.Now()
+	duration := s.endTime.Sub(s.startTime)
+	fmt.Printf("[TRACE] Span ended: name=%s duration=%v status=%s\n",
+		s.name, duration, s.status)
+}
+
+func (s *SimpleSpan) SetAttributes(attrs ...retry.Attribute) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.attributes = append(s.attributes, attrs...)
+}
+
+func (s *SimpleSpan) SetStatus(code string, description string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.status = code
+	s.description = description
+}
+
+func (s *SimpleSpan) AddEvent(name string, attrs ...retry.Attribute) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	event := SimpleEvent{
+		name:       name,
+		timestamp:  time.Now(),
+		attributes: attrs,
+	}
+	s.events = append(s.events, event)
+	fmt.Printf("[TRACE] Event added: span=%s event=%s\n", s.name, name)
+}
+
+func main() {
+	// Create observability components
+	metrics := &SimpleMetricsCollector{}
+	tracer := &SimpleTracer{}
+	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	}))
+
+	// Create a test server that fails twice then succeeds
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		// Simulate processing time
+		time.Sleep(10 * time.Millisecond)
+
+		if attempts < 3 {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprintf(w, "Temporary failure (attempt %d)", attempts)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "Success!")
+	}))
+	defer server.Close()
+
+	// Create retry client with all observability features
+	client, err := retry.NewClient(
+		retry.WithMaxRetries(5),
+		retry.WithInitialRetryDelay(50*time.Millisecond),
+		retry.WithJitter(false), // Disable for predictable demo output
+		retry.WithMetrics(metrics),
+		retry.WithTracer(tracer),
+		retry.WithLogger(retry.NewSlogAdapter(logger)),
+	)
+	if err != nil {
+		logger.Error("failed to create client", "error", err)
+		os.Exit(1)
+	}
+
+	// Make request
+	ctx := context.Background()
+	fmt.Println("=== Starting HTTP request with full observability ===\n")
+
+	resp, err := client.Get(ctx, server.URL)
+	if err != nil {
+		logger.Error("request failed", "error", err)
+		os.Exit(1)
+	}
+	defer resp.Body.Close()
+
+	fmt.Println("\n=== Request completed successfully ===")
+	fmt.Printf("Status: %d\n", resp.StatusCode)
+	fmt.Printf("Total Attempts: %d\n", attempts)
+
+	// Print metrics summary
+	metrics.PrintSummary()
+
+	fmt.Println("\n=== TRACE SUMMARY ===")
+	fmt.Printf("Total Spans Created: %d\n", len(tracer.spans))
+}

_example/observability/opentelemetry/main.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	retry "github.com/appleboy/go-httpretry"
+)
+
+// OTelTracer implements retry.Tracer for OpenTelemetry
+// This is a simplified example showing the interface implementation.
+// In production, you would use go.opentelemetry.io/otel
+type OTelTracer struct {
+	serviceName string
+}
+
+func NewOTelTracer(serviceName string) *OTelTracer {
+	return &OTelTracer{serviceName: serviceName}
+}
+
+func (t *OTelTracer) StartSpan(ctx context.Context, operationName string, attrs ...retry.Attribute) (context.Context, retry.Span) {
+	// In production:
+	// tracer := otel.Tracer(t.serviceName)
+	// ctx, span := tracer.Start(ctx, operationName)
+	// for _, attr := range attrs {
+	//     span.SetAttributes(attribute.String(attr.Key, fmt.Sprint(attr.Value)))
+	// }
+	// return ctx, &OTelSpan{span: span}
+
+	span := &OTelSpan{
+		name:      operationName,
+		startTime: time.Now(),
+	}
+
+	fmt.Printf("[OpenTelemetry] Starting span: %s\n", operationName)
+	for _, attr := range attrs {
+		fmt.Printf("  - %s = %v\n", attr.Key, attr.Value)
+	}
+
+	return ctx, span
+}
+
+// OTelSpan wraps an OpenTelemetry span
+type OTelSpan struct {
+	name      string
+	startTime time.Time
+	status    string
+	desc      string
+}
+
+func (s *OTelSpan) End() {
+	duration := time.Since(s.startTime)
+	// In production: s.span.End()
+	fmt.Printf("[OpenTelemetry] Ending span: %s (duration: %v, status: %s)\n", s.name, duration, s.status)
+}
+
+func (s *OTelSpan) SetAttributes(attrs ...retry.Attribute) {
+	// In production:
+	// for _, attr := range attrs {
+	//     s.span.SetAttributes(attribute.String(attr.Key, fmt.Sprint(attr.Value)))
+	// }
+
+	for _, attr := range attrs {
+		fmt.Printf("[OpenTelemetry] Setting attribute on %s: %s = %v\n", s.name, attr.Key, attr.Value)
+	}
+}
+
+func (s *OTelSpan) SetStatus(code string, description string) {
+	// In production:
+	// if code == "error" {
+	//     s.span.SetStatus(codes.Error, description)
+	// } else {
+	//     s.span.SetStatus(codes.Ok, description)
+	// }
+
+	s.status = code
+	s.desc = description
+	fmt.Printf("[OpenTelemetry] Setting status on %s: %s (%s)\n", s.name, code, description)
+}
+
+func (s *OTelSpan) AddEvent(name string, attrs ...retry.Attribute) {
+	// In production:
+	// eventAttrs := make([]attribute.KeyValue, len(attrs))
+	// for i, attr := range attrs {
+	//     eventAttrs[i] = attribute.String(attr.Key, fmt.Sprint(attr.Value))
+	// }
+	// s.span.AddEvent(name, trace.WithAttributes(eventAttrs...))
+
+	fmt.Printf("[OpenTelemetry] Adding event to %s: %s\n", s.name, name)
+	for _, attr := range attrs {
+		fmt.Printf("  - %s = %v\n", attr.Key, attr.Value)
+	}
+}
+
+func main() {
+	// In production, you would initialize OpenTelemetry:
+	// import (
+	//     "go.opentelemetry.io/otel"
+	//     "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	//     "go.opentelemetry.io/otel/sdk/trace"
+	// )
+	//
+	// exporter, _ := otlptrace.New(ctx, otlptrace.WithInsecure())
+	// tp := trace.NewTracerProvider(
+	//     trace.WithBatcher(exporter),
+	//     trace.WithResource(resource.NewWithAttributes(
+	//         semconv.SchemaURL,
+	//         semconv.ServiceName("my-service"),
+	//     )),
+	// )
+	// otel.SetTracerProvider(tp)
+
+	// Create OpenTelemetry tracer
+	otelTracer := NewOTelTracer("http-retry-service")
+
+	// Create test server that fails twice then succeeds
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		if attempts < 3 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	// Create retry client with OpenTelemetry tracing
+	client, err := retry.NewClient(
+		retry.WithMaxRetries(5),
+		retry.WithInitialRetryDelay(50*time.Millisecond),
+		retry.WithJitter(false),
+		retry.WithTracer(otelTracer),
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println("=== Making HTTP request with OpenTelemetry tracing ===\n")
+
+	// Make request
+	ctx := context.Background()
+	resp, err := client.Get(ctx, server.URL)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	fmt.Printf("\n=== Request completed: status=%d, attempts=%d ===\n", resp.StatusCode, attempts)
+
+	fmt.Println("\n// In production, these traces would be:")
+	fmt.Println("// - Exported to Jaeger, Zipkin, or other tracing backends")
+	fmt.Println("// - Visible in distributed tracing UI (e.g., Jaeger UI)")
+	fmt.Println("// - Showing the complete request flow with timing and dependencies")
+	fmt.Println("//")
+	fmt.Println("// Example trace structure:")
+	fmt.Println("// └─ http.retry.request (parent span)")
+	fmt.Println("//    ├─ http.retry.attempt (attempt 1) - failed")
+	fmt.Println("//    ├─ http.retry.attempt (attempt 2) - failed")
+	fmt.Println("//    └─ http.retry.attempt (attempt 3) - success")
+}