From cebf519d678d7a568c63bcf230d36d8f162a5c5b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2020 16:15:34 +0200 Subject: [PATCH] wip: support long CIDs in subdomains with TLS This is WIP Goal: add subdomain gateway support for CIDs longer than 63 characters in a way that fits in a single DNS label to enable TLS for every root. License: MIT Signed-off-by: Marcin Rataj --- core/corehttp/hostname.go | 59 +++++++++++++++++++---- core/corehttp/hostname_test.go | 41 +++++++++++++++- test/sharness/t0114-gateway-subdomains.sh | 40 ++++++++++++++- 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index 143435106e5..703b26f613c 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{ "dweb.link": subdomainGatewaySpec, } +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + // HostnameOption rewrites an incoming request based on the Host header. func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { childMux := http.NewServeMux() - coreApi, err := coreapi.NewCoreAPI(n) + coreAPI, err := coreapi.NewCoreAPI(n) if err != nil { return nil, err } @@ -124,7 +127,7 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) { + if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -151,16 +154,30 @@ func HostnameOption() ServeOption { return } - // Do we need to fix multicodec in PeerID represented as CIDv1? - if isPeerIDNamespace(ns) { - keyCid, err := cid.Decode(rootID) - if err == nil && keyCid.Type() != cid.Libp2pKey { - if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { - // Redirect to CID fixed inside of toSubdomainURL() + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect CID to a canonical DNS representation? + dnsID := toDNSPrefix(n.Context(), coreAPI, rootID) + if !strings.HasPrefix(r.Host, dnsID) { + dnsPrefix := "/" + ns + "/" + dnsID + if newURL, ok := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r); ok { + // Redirect to CID split split at deterministic places + // to ensure CID always gets the same Origin on the web http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } } // Rewrite the path to not use subdomains @@ -176,7 +193,7 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -266,6 +283,19 @@ func isPeerIDNamespace(ns string) bool { } } +// Converts an identifier to DNS-safe representation that fits in 63 characters +func toDNSPrefix(ctx context.Context, api iface.CoreAPI, id string) (prefix string) { + // Return as-is if things fit + if len(id) <= dnsLabelMaxLength { + return id + } + + // TODO: get original root + // TODO: create a new CID, representing a dns-safe root + + return id +} + // Converts a hostname/path to a subdomain-based URL, if applicable. func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { var scheme, ns, rootID, rest string @@ -340,6 +370,17 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok // produce a subdomain URL return "", false } + + // if IPNS rootID is too long, but Base36 makes it fit, switch to it + if len(rootID) > 63 && isPeerIDNamespace(ns) { + encoding, err := cid.ExtractEncoding(rootID) + if err == nil && encoding != mbase.Base36 { + rootIDb36, err := cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base36) + if err == nil && len(rootIDb36) <= 63 { + rootID = rootIDb36 + } + } + } } return safeRedirectURL(fmt.Sprintf( diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go index 9a297464891..411bd3ee880 100644 --- a/core/corehttp/hostname_test.go +++ b/core/corehttp/hostname_test.go @@ -23,11 +23,13 @@ func TestToSubdomainURL(t *testing.T) { {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, // CIDv0 → CIDv1base32 {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + // CIDv1 with long sha512 (requires DNS label length workaround) + {"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "http://bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvy.w764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg.ipfs.localhost/", true}, // PeerID as CIDv1 needs to have libp2p-key multicodec {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, // PeerID: ed25519+identity multihash - {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://ba.fzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, } { url, ok := toSubdomainURL(test.hostname, test.path, r) if ok != test.ok || url != test.url { @@ -75,6 +77,30 @@ func TestPortStripping(t *testing.T) { } +func TestDNSPrefix(t *testing.T) { + /* TODO + for _, test := range []struct { + in string + out string + }{ + // <= 63 + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm"}, + {"bafy.beickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm"}, + // > 63 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "TODO"}, + {"bafzaajaiaejca4syrpdu6g.dx4wsdnokxkprgzxf4wrs.tuc34gxw5k5jrag2so5gk", "TODO"}, + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "TODO"}, + {"bafkrgqe3ohjcjplc6n4f3fw.unlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4.ffyyxnayrtdi5oc4xb2332g645433aeg", "TODO"}, + } { + out := toDNSPrefix(test.in) + if out != test.out { + t.Errorf("(%s): returned '%s', expected '%s'", test.in, out, test.out) + } + } + */ + +} + func TestKnownSubdomainDetails(t *testing.T) { gwSpec := config.GatewaySpec{ UseSubdomains: true, @@ -127,6 +153,19 @@ func TestKnownSubdomainDetails(t *testing.T) { {"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false}, {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // edge case check: understand split CIDs (workaround for 63 character limit of a single DNS label https://github.com/ipfs/go-ipfs/issues/7318) + // Note: canonical split is at 63, but we support arbitrary splits for improved UX + // Short CID (eg. unnecessarily split by user) + /* TODO: long CID tests + {"baf.kreicysg23kiwv34eg2d7.qweipxwosdo2py4ldv4.2nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + // ED25519 libp2p-key + {"ba.fzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true}, + {"bafzaajaiaejca4syrpdu6gdx4wsdnok.xkprgzxf4wrstuc34gxw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true}, + {"bafzaajaiaejca4sy.rpdu6gdx4wsdnok.xkprgzxf4wrstuc34g.xw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true}, + // CID created with --hash sha2-512 + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq.6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true}, + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltg.gn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4f.fyyxnayrtdi5oc4xb2332g645433aeg.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true}, + */ // other namespaces {"api.localhost", "", "", "", false}, {"peerid.p2p.localhost", "localhost", "p2p", "peerid", true}, diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 8b5133259af..55da37fc288 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -92,6 +92,7 @@ test_launch_ipfs_daemon --offline test_expect_success "Add test text file" ' CID_VAL="hello" CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q) + CIDv1_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q) CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q) CIDv0to1=$(echo "$CIDv0" | ipfs cid base32) ' @@ -119,7 +120,6 @@ test_expect_success "Publish test text file to IPNS" ' test_cmp expected2 output ' - # ensure we start with empty Gateway.PublicGateways test_expect_success 'start daemon with empty config for Gateway.PublicGateways' ' test_kill_ipfs_daemon && @@ -262,6 +262,7 @@ test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/ test_should_contain "subdir2-bar" list_response ' + # *.ipns.localhost # .ipns.localhost @@ -501,6 +502,43 @@ test_hostname_gateway_response_should_contain \ "http://127.0.0.1:$GWAY_PORT" \ "404 Not Found" +## ============================================================================ +## Special handling of CIDs that do not fit in a single DNS Label (>63chars) +## https://github.com/ipfs/go-ipfs/issues/7318 +## ============================================================================ + +CID_DNS_TODO="TODO" + +# local: *.localhost +test_localhost_gateway_response_should_contain \ + "request for a long CID at localhost/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1_LONG" \ + "Location: http://${CID_DNS_TODO}.ipfs.localhost:$GWAY_PORT/" + +test_localhost_gateway_response_should_contain \ + "request for {dns-fixed-long-CID}.ipfs.localhost should return expected payload" \ + "http://${CID_DNS_TODO}.ipfs.localhost:$GWAY_PORT" \ + "$CID_VAL" + +# public gateway: *.example.com + +test_hostname_gateway_response_should_contain \ + "request for a long CID at example.com/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_LONG" \ + "Location: http://${CID_DNS_TODO}.ipfs.example.com" + +test_hostname_gateway_response_should_contain \ + "request for {long.CID}.ipfs.example.com should return expected payload" \ + "${CID_DNS_TODO}.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "$CID_VAL" + +test_hostname_gateway_response_should_contain \ + "request for {dns-fixed-long-CID}.ipfs.example.com should return redirect to a canonical Origin" \ + "${CID_DNS_TODO}.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "Location: http://${CID_DNS_TODO}.ipfs.example.com" ## ============================================================================ ## Test path-based requests with a custom hostname config