[BREAKING] feat: refactor Storage to store a complete *http.Response instead of just response bytes (#4)

Author: bored-engineer

bbolt/go.mod

@@ -2,6 +2,6 @@ module github.com/bored-engineer/github-conditional-http-transport/bbolt
 
 go 1.23.5
 
-require go.etcd.io/bbolt v1.3.11
+require go.etcd.io/bbolt v1.4.2
 
-require golang.org/x/sys v0.4.0 // indirect
+require golang.org/x/sys v0.33.0 // indirect

bbolt/go.sum

@@ -6,9 +6,13 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
+go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I=
+go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM=
 golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

bbolt/storage.go

@@ -1,10 +1,12 @@
 package bboltstorage
 
 import (
+	"bufio"
 	"bytes"
 	"context"
-	"io"
+	"fmt"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os"
 
@@ -17,34 +19,51 @@ type Storage struct {
 	Bucket []byte
 }
 
-func (s *Storage) Get(ctx context.Context, u *url.URL) (body io.ReadCloser, header http.Header, err error) {
+func (s *Storage) Get(ctx context.Context, u *url.URL) (*http.Response, error) {
+	var bodyBytes []byte
 	if err := s.DB.View(func(tx *bbolt.Tx) error {
 		bucket := tx.Bucket(s.Bucket)
 		if bucket == nil {
 			return bbolt.ErrBucketNotFound
 		}
-		bodyBytes := bucket.Get([]byte(u.String()))
-		if bodyBytes == nil {
+		bodyBytesUnsafe := bucket.Get([]byte(u.String()))
+		if bodyBytesUnsafe == nil {
 			return nil
 		}
-		bodyBytesSafe := make([]byte, len(bodyBytes))
-		copy(bodyBytesSafe, bodyBytes)
-		body = io.NopCloser(bytes.NewReader(bodyBytesSafe))
+		bodyBytes = make([]byte, len(bodyBytesUnsafe))
+		copy(bodyBytes, bodyBytesUnsafe)
 		return nil
 	}); err != nil {
-		return nil, nil, err
+		return nil, fmt.Errorf("(*bbolt.DB).View failed: %w", err)
 	}
-	return
+	if bodyBytes == nil {
+		return nil, nil
+	}
+	resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(bodyBytes)), nil)
+	if err != nil {
+		return nil, fmt.Errorf("http.ReadResponse failed: %w", err)
+	}
+	return resp, nil
 }
 
-func (s *Storage) Put(ctx context.Context, u *url.URL, body []byte, header http.Header) (err error) {
-	return s.DB.Update(func(tx *bbolt.Tx) error {
+func (s *Storage) Put(ctx context.Context, u *url.URL, resp *http.Response) error {
+	b, err := httputil.DumpResponse(resp, true)
+	if err != nil {
+		return fmt.Errorf("httputil.DumpResponse failed: %w", err)
+	}
+	if err := s.DB.Update(func(tx *bbolt.Tx) error {
 		bucket := tx.Bucket(s.Bucket)
 		if bucket == nil {
 			return bbolt.ErrBucketNotFound
 		}
-		return bucket.Put([]byte(u.String()), body)
-	})
+		if err := bucket.Put([]byte(u.String()), b); err != nil {
+			return fmt.Errorf("(*bbolt.Bucket).Put failed: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("(*bbolt.DB).Update failed: %w", err)
+	}
+	return nil
 }
 
 // Open is a wrapper around bbolt.Open that returns an initialized Storage.
@@ -54,13 +73,15 @@ func Open(path string, mode os.FileMode, options *bbolt.Options, bucket []byte)
 	}
 	db, err := bbolt.Open(path, mode, options)
 	if err != nil {
-		return &Storage{}, err
+		return &Storage{}, fmt.Errorf("bbolt.Open failed: %w", err)
 	}
 	if err := db.Update(func(tx *bbolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(bucket)
-		return err
+		if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
+			return fmt.Errorf("(*bbolt.Tx).CreateBucketIfNotExists failed: %w", err)
+		}
+		return nil
 	}); err != nil {
-		return &Storage{}, err
+		return &Storage{}, fmt.Errorf("(*bbolt.DB).Update failed: %w", err)
 	}
 	return &Storage{DB: db, Bucket: bucket}, nil
 }

hash.go

@@ -1,9 +1,12 @@
 package ghtransport
 
 import (
+	"bytes"
 	"crypto/sha256"
+	"encoding/base64"
 	"hash"
 	"net/http"
+	"strings"
 )
 
 // VaryHeaders are the headers that are used to vary the cache key, this slice _must_ remain sorted.
@@ -24,3 +27,24 @@ func Hash(requestHeaders http.Header) hash.Hash {
 	}
 	return h
 }
+
+// HashToken returns a hash of the 'Authorization' header matching the 'hashed_token' audit log field from GitHub.
+// https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/identifying-audit-log-events-performed-by-an-access-token
+func HashToken(authorization string) string {
+	// If the authorization header is empty, we hash an empty string
+	var token string
+	// This is the most common pattern
+	if bearer, ok := strings.CutPrefix(authorization, "Bearer "); ok && bearer != "" {
+		token = bearer
+	}
+	// This is the second most common pattern
+	if basic, ok := strings.CutPrefix(authorization, "Basic "); ok && basic != "" {
+		if decoded, err := base64.StdEncoding.DecodeString(basic); err == nil {
+			if _, password, ok := bytes.Cut(decoded, []byte{':'}); ok && len(password) > 0 {
+				token = string(password)
+			}
+		}
+	}
+	hashed := sha256.Sum256([]byte(token))
+	return base64.StdEncoding.EncodeToString(hashed[:])
+}

hash_test.go

@@ -0,0 +1,71 @@
+package ghtransport
+
+import (
+	"encoding/hex"
+	"net/http"
+	"testing"
+)
+
+const testBody = `{"login":"bored-engineer","id":541842,"node_id":"MDQ6VXNlcjU0MTg0Mg==","avatar_url":"https://avatars.githubusercontent.com/u/541842?v=4","gravatar_id":"","url":"https://api.github.com/users/bored-engineer","html_url":"https://github.com/bored-engineer","followers_url":"https://api.github.com/users/bored-engineer/followers","following_url":"https://api.github.com/users/bored-engineer/following{/other_user}","gists_url":"https://api.github.com/users/bored-engineer/gists{/gist_id}","starred_url":"https://api.github.com/users/bored-engineer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bored-engineer/subscriptions","organizations_url":"https://api.github.com/users/bored-engineer/orgs","repos_url":"https://api.github.com/users/bored-engineer/repos","events_url":"https://api.github.com/users/bored-engineer/events{/privacy}","received_events_url":"https://api.github.com/users/bored-engineer/received_events","type":"User","user_view_type":"public","site_admin":false,"name":"Luke Young","company":null,"blog":"https://bored.engineer/","location":"San Francisco, CA","email":null,"hireable":true,"bio":"I find bugs and exploit them. Sometimes for money, mainly for free T-Shirts...","twitter_username":null,"public_repos":136,"public_gists":51,"followers":212,"following":13,"created_at":"2010-12-30T17:15:38Z","updated_at":"2025-05-06T02:44:16Z"}`
+
+func TestHash(t *testing.T) {
+	tests := map[string]struct {
+		Headers  http.Header
+		Body     string
+		Expected string
+	}{
+		"emptyall": {
+			Headers:  http.Header{},
+			Body:     "",
+			Expected: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+		},
+		"emptyheaders": {
+			Headers:  http.Header{},
+			Body:     testBody,
+			Expected: "c5542e3ee32c0adf1128a79d80a296d03412415c924e522d9d1c75b17d7c3ef0",
+		},
+		"accept": {
+			Headers: http.Header{
+				"Accept": []string{"application/vnd.github.v3+json"},
+			},
+			Body:     testBody,
+			Expected: "125f46f7d22cd8f41ea1534256ba85a45f4a0e3dcf995da9fecfe3361b93407d",
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			h := Hash(test.Headers)
+			h.Write([]byte(test.Body))
+			if got := hex.EncodeToString(h.Sum(nil)); got != test.Expected {
+				t.Errorf("Hash(%v, %q) = %x, want %x", test.Headers, test.Body, got, test.Expected)
+			}
+		})
+	}
+}
+
+func TestHashToken(t *testing.T) {
+	tests := map[string]struct {
+		Authorization string
+		Expected      string
+	}{
+		"empty": {
+			Authorization: "",
+			Expected:      "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
+		},
+		"bearer": {
+			Authorization: "Bearer hunter2",
+			Expected:      "9S+9MrKzuG/4jvbEkGKChfSCrxXdyylUH5S89Saj9sc=",
+		},
+		"basic": {
+			Authorization: "Basic Ym9yZWQtZW5naW5lZXI6aHVudGVyMg==",
+			Expected:      "9S+9MrKzuG/4jvbEkGKChfSCrxXdyylUH5S89Saj9sc=",
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := HashToken(test.Authorization); got != test.Expected {
+				t.Errorf("HashToken(%v) = %q, want %q", test.Authorization, got, test.Expected)
+			}
+		})
+	}
+}

internal/bufferpool/bufferpool.go

@@ -3,38 +3,59 @@ package memory
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"sync"
 )
 
+// cachedResponse wraps a *http.Response allowing the body bytes to be stored in memory.
+type cachedResponse struct {
+	Response *http.Response
+	Body     []byte
+}
+
 // Implements the ghtransport.Storage interface via a simple, un-bound in-memory map.
 type Storage struct {
 	lock *sync.RWMutex
-	m    map[string][]byte
+	m    map[string]cachedResponse
 }
 
-func (s Storage) Get(ctx context.Context, u *url.URL) (body io.ReadCloser, header http.Header, err error) {
+func (s Storage) Get(ctx context.Context, u *url.URL) (*http.Response, error) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
-	if body, ok := s.m[u.String()]; ok {
-		return io.NopCloser(bytes.NewReader(body)), nil, nil
+	body, ok := s.m[u.String()]
+	if !ok {
+		return nil, nil
 	}
-	return nil, nil, nil
+	resp := *body.Response
+	resp.Body = io.NopCloser(bytes.NewReader(body.Body))
+	return &resp, nil
 }
 
-func (s Storage) Put(ctx context.Context, u *url.URL, body []byte, header http.Header) (err error) {
+func (s Storage) Put(ctx context.Context, u *url.URL, resp *http.Response) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
-	s.m[u.String()] = body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("(*http.Response).Body.Read failed: %w", err)
+	}
+	if err := resp.Body.Close(); err != nil {
+		return fmt.Errorf("(*http.Response).Body.Close failed: %w", err)
+	}
+	resp.Body = nil
+	s.m[u.String()] = cachedResponse{
+		Response: resp,
+		Body:     body,
+	}
 	return nil
 }
 
 // NewStorage returns a new, empty Storage.
 func NewStorage() Storage {
 	return Storage{
 		lock: &sync.RWMutex{},
-		m:    make(map[string][]byte),
+		m:    make(map[string]cachedResponse),
 	}
 }

memory/storage.go

@@ -3,25 +3,25 @@ module github.com/bored-engineer/github-conditional-http-transport/s3
 go 1.23.5
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.36.0
-	github.com/aws/aws-sdk-go-v2/config v1.29.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2
+	github.com/aws/aws-sdk-go-v2 v1.36.5
+	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.57 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
-	github.com/aws/smithy-go v1.22.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/aws/smithy-go v1.22.4 // indirect
 )