From acb91b7a8682d79853ca817d5f16b0819c83d2f7 Mon Sep 17 00:00:00 2001 From: lxt2 Date: Wed, 5 Oct 2016 14:23:12 +1100 Subject: [PATCH] Refactor code to use http.RoundTripper --- README.md | 62 +++----------------- harhar/main.go | 73 +++++++---------------- client.go => recorder.go | 124 +++++++-------------------------------- structs.go | 45 ++++++++++++++ 4 files changed, 98 insertions(+), 206 deletions(-) rename client.go => recorder.go (55%) diff --git a/README.md b/README.md index a5aa02d..f6b6f57 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,20 @@ harhar ====== -Simple, transparent HTTP Archive (HAR) logging for Go code using the -http.Client interface. For existing code that already uses the `net/http` -package, updating existing code to generate HAR logs can typically be done -with only 2 lines of code. +HTTP Archive (HAR) recording for Go code using the http.RoundTripper interface. Getting Started --------------- -First, convert your existing http.Client instance (or http.DefaultClient) to -a harhar.Client: +For logging from an `http.Client` you can simply set the Transport property: - // before - webClient := &http.Client{} - - // after - httpClient := &http.Client{} - webClient := harhar.NewClient(httpClient) - -Then, whenever you're ready to generate the HAR output, call WriteLog: - - webClient.WriteLog("output.har") - -That's it! harhar.Client implements all the same methods as http.Client, so no -other code will need to be changed. However, if you set Timeouts, Cookies, etc. -dynamically then you will want to retain a copy of the wrapped http.Client. -harhar.Client only stores the pointer, so changes to the underlying http.Client -will be used immediately. - -Optional periodic logging -------------------------- - -To dynamically enable or disable HAR logging, code can use harhar.ClientInterface -to represent either an http.Client or harhar.Client. The included `harhar` example -command shows one way to use this interface. When using this interface, you can -write logs (if enabled) by using this simple block of code: - - if harCli, ok := myClient.(*harhar.Client); ok { - harCli.WriteLog("output.har") +```go + recorder := harhar.NewRecorder(http.DefaultTransport) + client := &http.Client{ + Transport: recorder, } +``` -When combined with a long-running process, the interface makes it possible to -toggle logging off and on, and periodically write to disk throughout a processes -lifetime. An example is the following (never-ending) goroutine: - - go func(){ - for _ = range time.Tick(time.Minute*5) { - if harCli, ok := myClient.(*harhar.Client); ok { - sz, err := harCli.WriteLog("output.har") - if err!=nil { - log.Println("error writing .har log:", err) - } else { - log.Printf("wrote .har log (%.1fkb)\n", float64(sz)/1024.0) - } - } - } - }() +Then, whenever you're ready to generate the HAR output, call WriteFile: -Note that when logging is enabled, harhar memory usage can grow pretty quickly, -especially if Responses are large. If you don't want to disable logging in code -when output size grows too large, you should at least display it so that users -can decide to stop before the OOM killer comes to play. + recorder.WriteFile("output.har") diff --git a/harhar/main.go b/harhar/main.go index bd70582..05488ca 100644 --- a/harhar/main.go +++ b/harhar/main.go @@ -1,75 +1,46 @@ -// This command will do a GET request on a provided list of URLs, optionally -// logging them to the HAR file. It's a simple example that concisely showcases -// all the features and usage. +// This command will do a GET request on a provided URL and log the result to a HAR file. +// It's a simple example that concisely showcases all the features and usage. package main import ( "flag" - "io/ioutil" "log" "net/http" "os" - "strings" - "github.com/stridatum/harhar" -) - -var ( - input = flag.String("urls", "", "input urls (one per line)") - output = flag.String("har", "", "output har to file") - - // client uses the interface just to show how it works. - // typically you'd use this so that you can toggle logging on an off - // at will to conserve memory usage. - client harhar.ClientInterface = &http.Client{} + harhar ".." ) func main() { + var ( + u = flag.String("url", "", "url to read") + output = flag.String("har", "", "output har to file") + ) + flag.Parse() - if *input == "" { + + if *u == "" || *output == "" { flag.Usage() os.Exit(1) } - if *output == "" { - log.Println("-har not provided, no .har file will be produced") - } else { - // wrap the http.Client to transparently track requests - client = harhar.NewClient(client.(*http.Client)) - } + recorder := harhar.NewRecorder() + client := &http.Client{Transport: recorder} - //////// - - // read in a file consisting of 1 line per URL, and do a GET on each. - data, err := ioutil.ReadFile(*input) + resp, err := client.Get(*u) if err != nil { - log.Println("error reading input: ", err) - os.Exit(1) - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - resp, err := client.Get(line) - if err != nil { - log.Println("error in GET to", line) - } else { - log.Printf("Got %s from %s\n", resp.Status, line) - } + log.Fatal(err) } - /////////// + log.Printf("got %s from %s\n", resp.Status, *u) - if *output != "" { - size, err := client.(*harhar.Client).WriteLog(*output) - if err == nil { - // it's always good to report size when logging since memory usage - // will grow pretty quickly if you're not careful. - log.Printf("wrote %s (%.1fkb)\n", *output, float64(size)/1024.0) - } else { - log.Println("error writing har: ", err) - } + size, err := recorder.WriteFile(*output) + if err != nil { + log.Fatal(err) } + + // it's always good to report size when logging since memory usage + // will grow pretty quickly if you're not careful. + log.Printf("wrote %s (%.1fkb)\n", *output, float64(size)/1024.0) } diff --git a/client.go b/recorder.go similarity index 55% rename from client.go rename to recorder.go index 883225a..c2d2004 100644 --- a/client.go +++ b/recorder.go @@ -29,117 +29,41 @@ package harhar import ( "bytes" "encoding/json" - "io" "io/ioutil" "net/http" - "net/url" "os" - "strings" "time" ) -// Client embeds a http.Client and wraps its methods to perform transparent HAR -// logging for every request and response. It contains the properties for the -// root "log" node of the HAR, with Creator, Version, and Comment strings. -type Client struct { - cli *http.Client - - // Creator describes the source of the logged requests/responses. - Creator struct { - // Name defaults to the name of the program (os.Args[0]) - Name string `json:"name"` - - // Version defaults to the current time (formatted as "20060102150405") - Version string `json:"version"` - } `json:"creator"` - - // Version defaults to the current time (formatted as "20060102150405") - Version string `json:"version"` - - // Comment can be added to the log to describe the particulars of this data. - Comment string `json:"comment,omitempty"` - - // Entries contains all of the Request and Response details that passed - // through this Client. - Entries []Entry `json:"entries"` +// Client embeds an upstream RoundTripper and wraps its methods to perform transparent HAR +// logging for every request and response +type Recorder struct { + http.RoundTripper `json:"-"` + HAR *HAR } -// ClientInterface allows you to dynamically swap in a harhar.Client for -// a http.Client if needed. (Although you'll still need to type-convert to use -// http.Client fields or WriteLog) -type ClientInterface interface { - Get(url string) (*http.Response, error) - Head(url string) (*http.Response, error) - Post(url string, bodyType string, body io.Reader) (*http.Response, error) - PostForm(url string, data url.Values) (*http.Response, error) - Do(req *http.Request) (*http.Response, error) -} +// NewRecorder returns a new Recorder object that fulfills the http.RoundTripper interface +func NewRecorder() *Recorder { + h := NewHAR() + h.Log.Creator.Name = os.Args[0] -func NewClient(client *http.Client) *Client { - nowVersion := time.Now().Format("20060102150405") - c := &Client{ - cli: client, - Version: nowVersion, + return &Recorder{ + RoundTripper: http.DefaultTransport, + HAR: h, } - // add some reasonable defaults - c.Creator.Name = os.Args[0] - c.Creator.Version = nowVersion - return c } // WriteLog writes the HAR log format to the filename given, then returns the // number of bytes. -func (c *Client) WriteLog(filename string) (int, error) { - data, err := json.Marshal(map[string]*Client{"log": c}) +func (c *Recorder) WriteFile(filename string) (int, error) { + data, err := json.Marshal(c.HAR) if err != nil { return 0, err } return len(data), ioutil.WriteFile(filename, data, 0644) } -/////////////////////////// -// wrappers to implement same interface as http.Client -/////////////////////////// - -// Get works just like http.Client.Get, creating a GET Request and calling Do. -func (c *Client) Get(url string) (*http.Response, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - return c.Do(req) -} - -// Head works just like http.Client.Head, creating a HEAD Request and calling Do. -func (c *Client) Head(url string) (*http.Response, error) { - req, err := http.NewRequest("HEAD", url, nil) - if err != nil { - return nil, err - } - return c.Do(req) -} - -// Post works just like http.Client.Post, creating a POST Request with the -// provided body data, setting the content-type to bodyType, and calling Do. -func (c *Client) Post(url string, bodyType string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest("POST", url, body) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", bodyType) - return c.Do(req) -} - -// PostForm works just like http.Client.PostForm, creating a POST Request by -// urlencoding data, setting the content-type appropriately, and calling Do. -func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) { - return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) -} - -// Do works by calling http.Client.Do on the wrapped client instance. However, -// it also tracks the request start and end times, and parses elements from the -// request and response data into HAR log Entries. -func (c *Client) Do(req *http.Request) (*http.Response, error) { +func (c *Recorder) RoundTrip(req *http.Request) (*http.Response, error) { var err error ent := Entry{} ent.Request, err = makeRequest(req) @@ -149,24 +73,22 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) { ent.Cache = make(map[string]string) startTime := time.Now() - resp, err := c.cli.Do(req) - finish := time.Now() + resp, err := c.RoundTripper.RoundTrip(req) if err != nil { return resp, err } - // very hard to get these numbers + ent.Timings.Wait = int(time.Now().Sub(startTime).Seconds() * 1000.0) + ent.Time = ent.Timings.Wait + + // TODO: implement send and receive ent.Timings.Send = -1 ent.Timings.Receive = -1 - ent.Timings.Wait = int(finish.Sub(startTime).Seconds() * 1000.0) - ent.Time = ent.Timings.Wait - ent.Start = startTime.Format(time.RFC3339Nano) ent.Response, err = makeResponse(resp) - // add entry to log - c.Entries = append(c.Entries, ent) + c.HAR.Log.Entries = append(c.HAR.Log.Entries, ent) return resp, err } @@ -238,7 +160,7 @@ func makeRequest(hr *http.Request) (Request, error) { func makeResponse(hr *http.Response) (Response, error) { r := Response{ StatusCode: hr.StatusCode, - StatusText: hr.Status[4:], // "200 OK" => "OK" + StatusText: http.StatusText(hr.StatusCode), HttpVersion: hr.Proto, HeadersSize: -1, BodySize: -1, @@ -267,8 +189,6 @@ func makeResponse(hr *http.Response) (Response, error) { r.Cookies = append(r.Cookies, nc) } - // TODO: check for redirect URL? - // read in all the data and replace the ReadCloser bodyData, err := ioutil.ReadAll(hr.Body) if err != nil { diff --git a/structs.go b/structs.go index 32fd51e..7457e16 100644 --- a/structs.go +++ b/structs.go @@ -24,6 +24,8 @@ THE SOFTWARE. package harhar +import "time" + // This file contains the struct definitinos for the various components of a // HAR logfile. It omits many optional properties for brevity, and because // harhar is generally only useful in a server (non-browser) application mode. @@ -31,6 +33,49 @@ package harhar // W3C Spec: // https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html +type HAR struct { + Log Log `json:"log"` +} + +func NewHAR() *HAR { + v := time.Now().Format("20060102150405") + + return &HAR{ + Log: Log{ + Version: v, + Creator: Creator{ + Version: v, + }, + }, + } + // add some reasonable defaults + // r.Creator.Name = os.Args[0] + // r.Creator.Version = nowVersion +} + +// Creator describes the source of the logged requests/responses. +type Creator struct { + // Name defaults to the name of the program (os.Args[0]) + Name string `json:"name"` + + // Version defaults to the current time (formatted as "20060102150405") + Version string `json:"version"` +} + +type Log struct { + Creator Creator `json:"creator"` + + // Version defaults to the current time (formatted as "20060102150405") + Version string `json:"version"` + + // Comment can be added to the log to describe the particulars of this data. + Comment string `json:"comment,omitempty"` + + // Entries contains all of the Request and Response details that passed + // through this Client. + Entries []Entry `json:"entries"` +} + type Entry struct { Request Request `json:"request"` Response Response `json:"response"`