feat: cache resolved refs, improve URI reader extensibility

This improves extensibility of how URI references are read when loading
OpenAPI 3 documents, and introduces simple URI-based caching.

One could, for example, integate new URI readers, such as one backed by
embed.FS.

A basic map-based cache implementation is provided, which may cover many
simple use cases. The cache interface introduced here may be implemented
with a third-party backend such as an LRU cache or Redis for more
advanced use cases.

RFC 7234 HTTP caching may also be implemented by customizing the HTTP
client used to read remote URI references. Customizing the HTTP client
also allows further customization of the transport, timeouts, etc.

Author: cmars

openapi3/loader.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -31,7 +29,7 @@ type Loader struct {
 	IsExternalRefsAllowed bool
 
 	// ReadFromURIFunc allows overriding the any file/URL reading func
-	ReadFromURIFunc func(loader *Loader, url *url.URL) ([]byte, error)
+	ReadFromURIFunc ReadFromURIFunc
 
 	Context context.Context
 
@@ -121,22 +119,7 @@ func (loader *Loader) readURL(location *url.URL) ([]byte, error) {
 	if f := loader.ReadFromURIFunc; f != nil {
 		return f(loader, location)
 	}
-
-	if location.Scheme != "" && location.Host != "" {
-		resp, err := http.Get(location.String())
-		if err != nil {
-			return nil, err
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode > 399 {
-			return nil, fmt.Errorf("error loading %q: request returned status code %d", location.String(), resp.StatusCode)
-		}
-		return ioutil.ReadAll(resp.Body)
-	}
-	if location.Scheme != "" || location.Host != "" || location.RawQuery != "" {
-		return nil, fmt.Errorf("unsupported URI: %q", location.String())
-	}
-	return ioutil.ReadFile(location.Path)
+	return DefaultReadFromURI(loader, location)
 }
 
 // LoadFromData loads a spec from a byte array

openapi3/loader_uri_reader.go

@@ -0,0 +1,115 @@
+package openapi3
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+)
+
+// ReadFromURIFunc defines a function which reads the contents of a resource
+// located at a URI.
+type ReadFromURIFunc func(loader *Loader, url *url.URL) ([]byte, error)
+
+// ErrURINotSupported indicates the ReadFromURIFunc does not know how to handle a
+// given URI.
+var ErrURINotSupported = fmt.Errorf("unsupported URI")
+
+// ReadFromURIs returns a ReadFromURIFunc which tries to read a URI using the
+// given reader functions, in the same order. If a reader function does not
+// support the URI and returns ErrURINotSupported, the next function is checked
+// until a match is found, or the URI is not supported by any.
+func ReadFromURIs(readers ...ReadFromURIFunc) ReadFromURIFunc {
+	return func(loader *Loader, url *url.URL) ([]byte, error) {
+		for i := range readers {
+			buf, err := readers[i](loader, url)
+			if err == ErrURINotSupported {
+				continue
+			} else if err != nil {
+				return nil, err
+			}
+			return buf, nil
+		}
+		return nil, ErrURINotSupported
+	}
+}
+
+// DefaultReadFromURI returns a ReadFromURIFunc which can read remote HTTP URIs and
+// local file URIs.
+var DefaultReadFromURI = ReadFromURIs(ReadFromHTTP(http.DefaultClient), ReadFromFile)
+
+// ReadFromHTTP returns a ReadFromURIFunc which uses the given http.Client to
+// read the contents from a remote HTTP URI. This client may be customized to
+// implement timeouts, RFC 7234 caching, etc.
+func ReadFromHTTP(cl *http.Client) ReadFromURIFunc {
+	return func(loader *Loader, location *url.URL) ([]byte, error) {
+		if location.Scheme == "" || location.Host == "" {
+			return nil, ErrURINotSupported
+		}
+		req, err := http.NewRequest("GET", location.String(), nil)
+		if err != nil {
+			return nil, err
+		}
+		resp, err := cl.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode > 399 {
+			return nil, fmt.Errorf("error loading %q: request returned status code %d", location.String(), resp.StatusCode)
+		}
+		return ioutil.ReadAll(resp.Body)
+	}
+}
+
+// ReadFromFile is a ReadFromURIFunc which reads local file URIs.
+func ReadFromFile(loader *Loader, location *url.URL) ([]byte, error) {
+	if location.Host != "" {
+		return nil, ErrURINotSupported
+	}
+	if location.Scheme != "" && location.Scheme != "file" {
+		return nil, ErrURINotSupported
+	}
+	return ioutil.ReadFile(location.Path)
+}
+
+// URIReadCache defines a cache for the contents read from URI references.
+type URIReadCache interface {
+	Get(location *url.URL) ([]byte, bool)
+	Set(location *url.URL, contents []byte)
+}
+
+// MapURIReadCache implements URIReadCache with a simple map.
+type MapURIReadCache map[string][]byte
+
+// Get implements URIReadCache.
+func (m MapURIReadCache) Get(location *url.URL) ([]byte, bool) {
+	contents, ok := m[location.String()]
+	return contents, ok
+}
+
+// Set implements URIReadCache.
+func (m MapURIReadCache) Set(location *url.URL, contents []byte) {
+	m[location.String()] = contents
+}
+
+// ReadFromCache returns a cached ReadFromURIFunc. If cache is nil, a new
+// internal cache is allocated, scoped to the given reader.
+func ReadFromCache(cache URIReadCache, r ReadFromURIFunc) ReadFromURIFunc {
+	if cache == nil {
+		cache = MapURIReadCache{}
+	}
+	return func(loader *Loader, location *url.URL) ([]byte, error) {
+		var err error
+		cached, ok := cache.Get(location)
+		if ok {
+			return cached, nil
+		}
+		cached, err = r(loader, location)
+		if err != nil {
+			return nil, err
+		}
+		cache.Set(location, cached)
+		return cached, nil
+	}
+}