From ba2b3b6807e1fec9e0b2ea11fbf2532f2f0f9e22 Mon Sep 17 00:00:00 2001 From: Kunihiko Sakamoto Date: Fri, 29 Mar 2019 11:17:19 +0900 Subject: [PATCH] go/bundle: Add support for the manifest section (#414) --- go/bundle/bundle.go | 107 ++++++++++++++++++++++------ go/bundle/cmd/dump-bundle/main.go | 4 ++ go/bundle/cmd/gen-bundle/fromdir.go | 11 ++- go/bundle/cmd/gen-bundle/main.go | 13 ++-- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/go/bundle/bundle.go b/go/bundle/bundle.go index 7ce23e32..a8ac7a47 100644 --- a/go/bundle/bundle.go +++ b/go/bundle/bundle.go @@ -118,7 +118,8 @@ func (e *Exchange) Dump(w io.Writer, dumpContentText bool) error { } type Bundle struct { - Exchanges []*Exchange + Exchanges []*Exchange + ManifestURL *url.URL } var _ = io.WriterTo(&Bundle{}) @@ -138,6 +139,12 @@ type requestEntryWithOffset struct { Offset uint64 } +type section interface { + Name() string + Len() int + io.WriterTo +} + // staging area for writing index section type indexSection struct { es []requestEntry @@ -210,6 +217,10 @@ func (is *indexSection) Finalize() error { return nil } +func (is *indexSection) Name() string { + return "index" +} + func (is *indexSection) Len() int { if is.bytes == nil { panic("indexSection must be Finalize()-d before calling Len()") @@ -217,11 +228,12 @@ func (is *indexSection) Len() int { return len(is.bytes) } -func (is *indexSection) Bytes() []byte { +func (is *indexSection) WriteTo(w io.Writer) (int64, error) { if is.bytes == nil { panic("indexSection must be Finalize()-d before calling Bytes()") } - return is.bytes + n, err := w.Write(is.bytes) + return int64(n), err } // staging area for writing responses section @@ -263,8 +275,26 @@ func (rs *responsesSection) addResponse(r Response) (int, error) { return length, nil } -func (rs *responsesSection) Len() int { return rs.buf.Len() } -func (rs *responsesSection) Bytes() []byte { return rs.buf.Bytes() } +func (rs *responsesSection) Name() string { return "responses" } +func (rs *responsesSection) Len() int { return rs.buf.Len() } +func (rs *responsesSection) WriteTo(w io.Writer) (int64, error) { + return rs.buf.WriteTo(w) +} + +type manifestSection struct { + bytes.Buffer +} + +func (ms *manifestSection) Name() string { return "manifest" } + +func newManifestSection(url *url.URL) (*manifestSection, error) { + var ms manifestSection + enc := cbor.NewEncoder(&ms) + if err := enc.EncodeTextString(url.String()); err != nil { + return nil, err + } + return &ms, nil +} func addExchange(is *indexSection, rs *responsesSection, e *Exchange) error { length, err := rs.addResponse(e.Response) @@ -296,17 +326,17 @@ func FindSection(sos []sectionOffset, name string) (sectionOffset, uint64, bool) // https://wicg.github.io/webpackage/draft-yasskin-dispatch-bundled-exchanges.html#load-metadata // Steps 3-7. -func writeSectionOffsets(w io.Writer, sos []sectionOffset) error { +func writeSectionOffsets(w io.Writer, sections []section) error { var b bytes.Buffer nenc := cbor.NewEncoder(&b) - if err := nenc.EncodeArrayHeader(len(sos) * 2); err != nil { + if err := nenc.EncodeArrayHeader(len(sections) * 2); err != nil { return err } - for _, so := range sos { - if err := nenc.EncodeTextString(so.Name); err != nil { + for _, s := range sections { + if err := nenc.EncodeTextString(s.Name()); err != nil { return err } - if err := nenc.EncodeUint(so.Length); err != nil { + if err := nenc.EncodeUint(uint64(s.Len())); err != nil { return err } } @@ -346,6 +376,7 @@ func writeFooter(w io.Writer, offset int) error { type meta struct { sectionOffsets []sectionOffset sectionsStart uint64 + manifestURL *url.URL requests []requestEntryWithOffset } @@ -578,8 +609,31 @@ func parseIndexSection(sectionContents []byte, sectionsStart uint64, sos []secti return requests, nil } +// https://wicg.github.io/webpackage/draft-yasskin-wpack-bundled-exchanges.html#manifest-section +// "To parse the manifest section, given its sectionContents and the metadata map to fill in, the parser MUST do the following:" [spec text] +func parseManifestSection(sectionContents []byte) (*url.URL, error) { + // Step 1. "Let urlString be the result of parsing sectionContents as a CBOR item matching the above manifest rule (Section 3.5). If urlString is an error, return that error." [spec text] + dec := cbor.NewDecoder(bytes.NewBuffer(sectionContents)) + urlString, err := dec.DecodeTextString() + if err != nil { + return nil, fmt.Errorf("bundle: failed to parse manifest section: %v", err) + } + // Step 2. "Let url be the result of parsing ([URL]) urlString with no base URL." [spec text] + manifestURL, err := url.Parse(urlString) + // Step 3. "If url is a failure, its fragment is not null, or it includes credentials, return an error." [spec text] + if err != nil { + return nil, fmt.Errorf("bundle: failed to parse manifest URL (%s): %v", urlString, err) + } + if !manifestURL.IsAbs() || manifestURL.Fragment != "" || manifestURL.User != nil { + return nil, fmt.Errorf("bundle: manifest URL (%s) must be an absolute url without fragment or credentials.", urlString) + } + // Step 4. "Set metadata["manifest"] to url." [spec text] + return manifestURL, nil +} + var knownSections = map[string]struct{}{ "index": struct{}{}, + "manifest": struct{}{}, "responses": struct{}{}, } @@ -694,6 +748,12 @@ func loadMetadata(bs []byte) (*meta, error) { return nil, err } meta.requests = requests + case "manifest": + manifestURL, err := parseManifestSection(sectionContents) + if err != nil { + return nil, err + } + meta.manifestURL = manifestURL case "responses": continue default: @@ -817,7 +877,7 @@ func Read(r io.Reader) (*Bundle, error) { es = append(es, e) } - b := &Bundle{Exchanges: es} + b := &Bundle{Exchanges: es, ManifestURL: m.manifestURL} return b, nil } @@ -836,25 +896,30 @@ func (b *Bundle) WriteTo(w io.Writer) (int64, error) { return cw.Written, err } - sos := []sectionOffset{ - sectionOffset{"index", uint64(is.Len())}, - sectionOffset{"responses", uint64(rs.Len())}, + sections := []section{} + sections = append(sections, is) + if b.ManifestURL != nil { + ms, err := newManifestSection(b.ManifestURL) + if err != nil { + return cw.Written, err + } + sections = append(sections, ms) } + sections = append(sections, rs) if _, err := cw.Write(HeaderMagicBytes); err != nil { return cw.Written, err } - if err := writeSectionOffsets(cw, sos); err != nil { - return cw.Written, err - } - if err := writeSectionHeader(cw, len(sos)); err != nil { + if err := writeSectionOffsets(cw, sections); err != nil { return cw.Written, err } - if _, err := cw.Write(is.Bytes()); err != nil { + if err := writeSectionHeader(cw, len(sections)); err != nil { return cw.Written, err } - if _, err := cw.Write(rs.Bytes()); err != nil { - return cw.Written, err + for _, s := range sections { + if _, err := s.WriteTo(cw); err != nil { + return cw.Written, err + } } if err := writeFooter(cw, int(cw.Written)); err != nil { return cw.Written, err diff --git a/go/bundle/cmd/dump-bundle/main.go b/go/bundle/cmd/dump-bundle/main.go index 821e17c1..459cd9e3 100644 --- a/go/bundle/cmd/dump-bundle/main.go +++ b/go/bundle/cmd/dump-bundle/main.go @@ -29,6 +29,10 @@ func run() error { return err } + if b.ManifestURL != nil { + fmt.Printf("Manifest URL: %v\n", b.ManifestURL) + } + for _, e := range b.Exchanges { if err := e.Dump(os.Stdout, *flagDumpContentText); err != nil { return err diff --git a/go/bundle/cmd/gen-bundle/fromdir.go b/go/bundle/cmd/gen-bundle/fromdir.go index e6840f04..e1f75e79 100644 --- a/go/bundle/cmd/gen-bundle/fromdir.go +++ b/go/bundle/cmd/gen-bundle/fromdir.go @@ -13,7 +13,7 @@ import ( "github.com/WICG/webpackage/go/bundle" ) -func fromDir(dir string, baseURL string, startURL string) error { +func fromDir(dir string, baseURL string, startURL string, manifestURL string) error { parsedBaseURL, err := url.Parse(baseURL) if err != nil { return fmt.Errorf("Failed to parse base URL. err: %v", err) @@ -22,6 +22,13 @@ func fromDir(dir string, baseURL string, startURL string) error { if err != nil { return fmt.Errorf("Failed to parse start URL. err: %v", err) } + var parsedManifestURL *url.URL + if len(manifestURL) > 0 { + parsedManifestURL, err = parsedBaseURL.Parse(manifestURL) + if err != nil { + return fmt.Errorf("Failed to parse manifest URL. err: %v", err) + } + } fo, err := os.OpenFile(*flagOutput, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { @@ -33,7 +40,7 @@ func fromDir(dir string, baseURL string, startURL string) error { if err != nil { return err } - b := &bundle.Bundle{Exchanges: es} + b := &bundle.Bundle{Exchanges: es, ManifestURL: parsedManifestURL} // Move the startURL entry to first. for i, e := range b.Exchanges { if e.Request.URL.String() == parsedStartURL.String() { diff --git a/go/bundle/cmd/gen-bundle/main.go b/go/bundle/cmd/gen-bundle/main.go index f3442362..a0b2b488 100644 --- a/go/bundle/cmd/gen-bundle/main.go +++ b/go/bundle/cmd/gen-bundle/main.go @@ -19,11 +19,12 @@ import ( ) var ( - flagHar = flag.String("har", "", "HTTP Archive (HAR) input file") - flagDir = flag.String("dir", "", "Input directory") - flagBaseURL = flag.String("baseURL", "", "Base URL") - flagStartURL = flag.String("startURL", "", "Entry point URL (relative from -baseURL)") - flagOutput = flag.String("o", "out.wbn", "Webbundle output file") + flagHar = flag.String("har", "", "HTTP Archive (HAR) input file") + flagDir = flag.String("dir", "", "Input directory") + flagBaseURL = flag.String("baseURL", "", "Base URL") + flagStartURL = flag.String("startURL", "", "Entry point URL (relative from -baseURL)") + flagManifestURL = flag.String("manifestURL", "", "Manifest URL (relative from -baseURL)") + flagOutput = flag.String("o", "out.wbn", "Webbundle output file") ) func ReadHar(r io.Reader) (*hargo.Har, error) { @@ -150,7 +151,7 @@ func main() { flag.Usage() return } - if err := fromDir(*flagDir, *flagBaseURL, *flagStartURL); err != nil { + if err := fromDir(*flagDir, *flagBaseURL, *flagStartURL, *flagManifestURL); err != nil { log.Fatal(err) } } else {