Skip to content

Commit

Permalink
Merge pull request pbnjay#1 from cardigann/http-roundtripper-refactor
Browse files Browse the repository at this point in the history 
Refactor code to use http.RoundTripper
  • Loading branch information
@pbnjay
pbnjay committed Nov 1, 2023
2 parents ff0c349 + acb91b7 commit a3d0b0a
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 206 deletions.
62 changes: 9 additions & 53 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,20 @@
harhar
======

Simple, transparent HTTP Archive (HAR) logging for Go code using the
http.Client interface. For existing code that already uses the `net/http`
package, updating existing code to generate HAR logs can typically be done
with only 2 lines of code.
HTTP Archive (HAR) recording for Go code using the http.RoundTripper interface.

Getting Started
---------------

First, convert your existing http.Client instance (or http.DefaultClient) to
a harhar.Client:
For logging from an `http.Client` you can simply set the Transport property:

// before
webClient := &http.Client{}

// after
httpClient := &http.Client{}
webClient := harhar.NewClient(httpClient)

Then, whenever you're ready to generate the HAR output, call WriteLog:

webClient.WriteLog("output.har")

That's it! harhar.Client implements all the same methods as http.Client, so no
other code will need to be changed. However, if you set Timeouts, Cookies, etc.
dynamically then you will want to retain a copy of the wrapped http.Client.
harhar.Client only stores the pointer, so changes to the underlying http.Client
will be used immediately.

Optional periodic logging
-------------------------

To dynamically enable or disable HAR logging, code can use harhar.ClientInterface
to represent either an http.Client or harhar.Client. The included `harhar` example
command shows one way to use this interface. When using this interface, you can
write logs (if enabled) by using this simple block of code:

if harCli, ok := myClient.(*harhar.Client); ok {
harCli.WriteLog("output.har")
```go
recorder := harhar.NewRecorder(http.DefaultTransport)
client := &http.Client{
Transport: recorder,
}
```

When combined with a long-running process, the interface makes it possible to
toggle logging off and on, and periodically write to disk throughout a processes
lifetime. An example is the following (never-ending) goroutine:

go func(){
for _ = range time.Tick(time.Minute*5) {
if harCli, ok := myClient.(*harhar.Client); ok {
sz, err := harCli.WriteLog("output.har")
if err!=nil {
log.Println("error writing .har log:", err)
} else {
log.Printf("wrote .har log (%.1fkb)\n", float64(sz)/1024.0)
}
}
}
}()
Then, whenever you're ready to generate the HAR output, call WriteFile:

Note that when logging is enabled, harhar memory usage can grow pretty quickly,
especially if Responses are large. If you don't want to disable logging in code
when output size grows too large, you should at least display it so that users
can decide to stop before the OOM killer comes to play.
recorder.WriteFile("output.har")
73 changes: 22 additions & 51 deletions harhar/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,75 +1,46 @@
// This command will do a GET request on a provided list of URLs, optionally
// logging them to the HAR file. It's a simple example that concisely showcases
// all the features and usage.
// This command will do a GET request on a provided URL and log the result to a HAR file.
// It's a simple example that concisely showcases all the features and usage.

package main

import (
"flag"
"io/ioutil"
"log"
"net/http"
"os"
"strings"

"github.com/stridatum/harhar"
)

var (
input = flag.String("urls", "", "input urls (one per line)")
output = flag.String("har", "", "output har to file")

// client uses the interface just to show how it works.
// typically you'd use this so that you can toggle logging on an off
// at will to conserve memory usage.
client harhar.ClientInterface = &http.Client{}
harhar ".."
)

func main() {
var (
u = flag.String("url", "", "url to read")
output = flag.String("har", "", "output har to file")
)

flag.Parse()
if *input == "" {

if *u == "" || *output == "" {
flag.Usage()
os.Exit(1)
}

if *output == "" {
log.Println("-har not provided, no .har file will be produced")
} else {
// wrap the http.Client to transparently track requests
client = harhar.NewClient(client.(*http.Client))
}
recorder := harhar.NewRecorder()
client := &http.Client{Transport: recorder}

////////

// read in a file consisting of 1 line per URL, and do a GET on each.
data, err := ioutil.ReadFile(*input)
resp, err := client.Get(*u)
if err != nil {
log.Println("error reading input: ", err)
os.Exit(1)
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
resp, err := client.Get(line)
if err != nil {
log.Println("error in GET to", line)
} else {
log.Printf("Got %s from %s\n", resp.Status, line)
}
log.Fatal(err)
}

///////////
log.Printf("got %s from %s\n", resp.Status, *u)

if *output != "" {
size, err := client.(*harhar.Client).WriteLog(*output)
if err == nil {
// it's always good to report size when logging since memory usage
// will grow pretty quickly if you're not careful.
log.Printf("wrote %s (%.1fkb)\n", *output, float64(size)/1024.0)
} else {
log.Println("error writing har: ", err)
}
size, err := recorder.WriteFile(*output)
if err != nil {
log.Fatal(err)
}

// it's always good to report size when logging since memory usage
// will grow pretty quickly if you're not careful.
log.Printf("wrote %s (%.1fkb)\n", *output, float64(size)/1024.0)
}
124 changes: 22 additions & 102 deletions client.go → recorder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,117 +29,41 @@ package harhar
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"time"
)

// Client embeds a http.Client and wraps its methods to perform transparent HAR
// logging for every request and response. It contains the properties for the
// root "log" node of the HAR, with Creator, Version, and Comment strings.
type Client struct {
cli *http.Client

// Creator describes the source of the logged requests/responses.
Creator struct {
// Name defaults to the name of the program (os.Args[0])
Name string `json:"name"`

// Version defaults to the current time (formatted as "20060102150405")
Version string `json:"version"`
} `json:"creator"`

// Version defaults to the current time (formatted as "20060102150405")
Version string `json:"version"`

// Comment can be added to the log to describe the particulars of this data.
Comment string `json:"comment,omitempty"`

// Entries contains all of the Request and Response details that passed
// through this Client.
Entries []Entry `json:"entries"`
// Client embeds an upstream RoundTripper and wraps its methods to perform transparent HAR
// logging for every request and response
type Recorder struct {
http.RoundTripper `json:"-"`
HAR *HAR
}

// ClientInterface allows you to dynamically swap in a harhar.Client for
// a http.Client if needed. (Although you'll still need to type-convert to use
// http.Client fields or WriteLog)
type ClientInterface interface {
Get(url string) (*http.Response, error)
Head(url string) (*http.Response, error)
Post(url string, bodyType string, body io.Reader) (*http.Response, error)
PostForm(url string, data url.Values) (*http.Response, error)
Do(req *http.Request) (*http.Response, error)
}
// NewRecorder returns a new Recorder object that fulfills the http.RoundTripper interface
func NewRecorder() *Recorder {
h := NewHAR()
h.Log.Creator.Name = os.Args[0]

func NewClient(client *http.Client) *Client {
nowVersion := time.Now().Format("20060102150405")
c := &Client{
cli: client,
Version: nowVersion,
return &Recorder{
RoundTripper: http.DefaultTransport,
HAR: h,
}
// add some reasonable defaults
c.Creator.Name = os.Args[0]
c.Creator.Version = nowVersion
return c
}

// WriteLog writes the HAR log format to the filename given, then returns the
// number of bytes.
func (c *Client) WriteLog(filename string) (int, error) {
data, err := json.Marshal(map[string]*Client{"log": c})
func (c *Recorder) WriteFile(filename string) (int, error) {
data, err := json.Marshal(c.HAR)
if err != nil {
return 0, err
}
return len(data), ioutil.WriteFile(filename, data, 0644)
}

///////////////////////////
// wrappers to implement same interface as http.Client
///////////////////////////

// Get works just like http.Client.Get, creating a GET Request and calling Do.
func (c *Client) Get(url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return c.Do(req)
}

// Head works just like http.Client.Head, creating a HEAD Request and calling Do.
func (c *Client) Head(url string) (*http.Response, error) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return nil, err
}
return c.Do(req)
}

// Post works just like http.Client.Post, creating a POST Request with the
// provided body data, setting the content-type to bodyType, and calling Do.
func (c *Client) Post(url string, bodyType string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return c.Do(req)
}

// PostForm works just like http.Client.PostForm, creating a POST Request by
// urlencoding data, setting the content-type appropriately, and calling Do.
func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) {
return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

// Do works by calling http.Client.Do on the wrapped client instance. However,
// it also tracks the request start and end times, and parses elements from the
// request and response data into HAR log Entries.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
func (c *Recorder) RoundTrip(req *http.Request) (*http.Response, error) {
var err error
ent := Entry{}
ent.Request, err = makeRequest(req)
Expand All @@ -149,24 +73,22 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
ent.Cache = make(map[string]string)

startTime := time.Now()
resp, err := c.cli.Do(req)
finish := time.Now()
resp, err := c.RoundTripper.RoundTrip(req)
if err != nil {
return resp, err
}

// very hard to get these numbers
ent.Timings.Wait = int(time.Now().Sub(startTime).Seconds() * 1000.0)
ent.Time = ent.Timings.Wait

// TODO: implement send and receive
ent.Timings.Send = -1
ent.Timings.Receive = -1

ent.Timings.Wait = int(finish.Sub(startTime).Seconds() * 1000.0)
ent.Time = ent.Timings.Wait

ent.Start = startTime.Format(time.RFC3339Nano)
ent.Response, err = makeResponse(resp)

// add entry to log
c.Entries = append(c.Entries, ent)
c.HAR.Log.Entries = append(c.HAR.Log.Entries, ent)
return resp, err
}

Expand Down Expand Up @@ -238,7 +160,7 @@ func makeRequest(hr *http.Request) (Request, error) {
func makeResponse(hr *http.Response) (Response, error) {
r := Response{
StatusCode: hr.StatusCode,
StatusText: hr.Status[4:], // "200 OK" => "OK"
StatusText: http.StatusText(hr.StatusCode),
HttpVersion: hr.Proto,
HeadersSize: -1,
BodySize: -1,
Expand Down Expand Up @@ -267,8 +189,6 @@ func makeResponse(hr *http.Response) (Response, error) {
r.Cookies = append(r.Cookies, nc)
}

// TODO: check for redirect URL?

// read in all the data and replace the ReadCloser
bodyData, err := ioutil.ReadAll(hr.Body)
if err != nil {
Expand Down
Loading

0 comments on commit a3d0b0a

Please sign in to comment.