Skip to content

Commit

Permalink
wip: support long CIDs in subdomains with TLS
Browse files Browse the repository at this point in the history 
This is WIP

Goal: add subdomain gateway support for CIDs longer than 63 characters
in a way that fits in a single DNS label to enable TLS for every root.

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>
  • Loading branch information
@lidel
lidel committed Jun 8, 2020
1 parent 11be1c5 commit cebf519
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 11 deletions.
59 changes: 50 additions & 9 deletions core/corehttp/hostname.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{
"dweb.link": subdomainGatewaySpec,
}

// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7)
const dnsLabelMaxLength int = 63

// HostnameOption rewrites an incoming request based on the Host header.
func HostnameOption() ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
childMux := http.NewServeMux()

coreApi, err := coreapi.NewCoreAPI(n)
coreAPI, err := coreapi.NewCoreAPI(n)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -124,7 +127,7 @@ func HostnameOption() ServeOption {
// Not a whitelisted path

// Try DNSLink, if it was not explicitly disabled for the hostname
if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) {
if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) {
// rewrite path and handle as DNSLink
r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path
childMux.ServeHTTP(w, r)
Expand All @@ -151,16 +154,30 @@ func HostnameOption() ServeOption {
return
}

// Do we need to fix multicodec in PeerID represented as CIDv1?
if isPeerIDNamespace(ns) {
keyCid, err := cid.Decode(rootID)
if err == nil && keyCid.Type() != cid.Libp2pKey {
if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok {
// Redirect to CID fixed inside of toSubdomainURL()
// Check if rootID is a valid CID
if rootCID, err := cid.Decode(rootID); err == nil {
// Do we need to redirect CID to a canonical DNS representation?
dnsID := toDNSPrefix(n.Context(), coreAPI, rootID)
if !strings.HasPrefix(r.Host, dnsID) {
dnsPrefix := "/" + ns + "/" + dnsID
if newURL, ok := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r); ok {
// Redirect to CID split split at deterministic places
// to ensure CID always gets the same Origin on the web
http.Redirect(w, r, newURL, http.StatusMovedPermanently)
return
}
}

// Do we need to fix multicodec in PeerID represented as CIDv1?
if isPeerIDNamespace(ns) {
if rootCID.Type() != cid.Libp2pKey {
if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok {
// Redirect to CID fixed inside of toSubdomainURL()
http.Redirect(w, r, newURL, http.StatusMovedPermanently)
return
}
}
}
}

// Rewrite the path to not use subdomains
Expand All @@ -176,7 +193,7 @@ func HostnameOption() ServeOption {
// 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)?
// 2. does Host header include a fully qualified domain name (FQDN)?
// 3. does DNSLink record exist in DNS?
if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) {
if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) {
// rewrite path and handle as DNSLink
r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path
childMux.ServeHTTP(w, r)
Expand Down Expand Up @@ -266,6 +283,19 @@ func isPeerIDNamespace(ns string) bool {
}
}

// Converts an identifier to DNS-safe representation that fits in 63 characters
func toDNSPrefix(ctx context.Context, api iface.CoreAPI, id string) (prefix string) {
// Return as-is if things fit
if len(id) <= dnsLabelMaxLength {
return id
}

// TODO: get original root
// TODO: create a new CID, representing a dns-safe root

return id
}

// Converts a hostname/path to a subdomain-based URL, if applicable.
func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) {
var scheme, ns, rootID, rest string
Expand Down Expand Up @@ -340,6 +370,17 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok
// produce a subdomain URL
return "", false
}

// if IPNS rootID is too long, but Base36 makes it fit, switch to it
if len(rootID) > 63 && isPeerIDNamespace(ns) {
encoding, err := cid.ExtractEncoding(rootID)
if err == nil && encoding != mbase.Base36 {
rootIDb36, err := cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base36)
if err == nil && len(rootIDb36) <= 63 {
rootID = rootIDb36
}
}
}
}

return safeRedirectURL(fmt.Sprintf(
Expand Down
41 changes: 40 additions & 1 deletion core/corehttp/hostname_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@ func TestToSubdomainURL(t *testing.T) {
{"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true},
// CIDv0 → CIDv1base32
{"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true},
// CIDv1 with long sha512 (requires DNS label length workaround)
{"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "http://bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvy.w764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg.ipfs.localhost/", true},
// PeerID as CIDv1 needs to have libp2p-key multicodec
{"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true},
{"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true},
// PeerID: ed25519+identity multihash
{"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true},
{"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://ba.fzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true},
} {
url, ok := toSubdomainURL(test.hostname, test.path, r)
if ok != test.ok || url != test.url {
Expand Down Expand Up @@ -75,6 +77,30 @@ func TestPortStripping(t *testing.T) {

}

func TestDNSPrefix(t *testing.T) {
/* TODO
for _, test := range []struct {
in string
out string
}{
// <= 63
{"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm"},
{"bafy.beickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm"},
// > 63
{"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "TODO"},
{"bafzaajaiaejca4syrpdu6g.dx4wsdnokxkprgzxf4wrs.tuc34gxw5k5jrag2so5gk", "TODO"},
{"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "TODO"},
{"bafkrgqe3ohjcjplc6n4f3fw.unlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4.ffyyxnayrtdi5oc4xb2332g645433aeg", "TODO"},
} {
out := toDNSPrefix(test.in)
if out != test.out {
t.Errorf("(%s): returned '%s', expected '%s'", test.in, out, test.out)
}
}
*/

}

func TestKnownSubdomainDetails(t *testing.T) {
gwSpec := config.GatewaySpec{
UseSubdomains: true,
Expand Down Expand Up @@ -127,6 +153,19 @@ func TestKnownSubdomainDetails(t *testing.T) {
{"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false},
{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
// edge case check: understand split CIDs (workaround for 63 character limit of a single DNS label https://github.com/ipfs/go-ipfs/issues/7318)
// Note: canonical split is at 63, but we support arbitrary splits for improved UX
// Short CID (eg. unnecessarily split by user)
/* TODO: long CID tests
{"baf.kreicysg23kiwv34eg2d7.qweipxwosdo2py4ldv4.2nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
// ED25519 libp2p-key
{"ba.fzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true},
{"bafzaajaiaejca4syrpdu6gdx4wsdnok.xkprgzxf4wrstuc34gxw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true},
{"bafzaajaiaejca4sy.rpdu6gdx4wsdnok.xkprgzxf4wrstuc34g.xw5k5jrag2so5gk.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true},
// CID created with --hash sha2-512
{"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq.6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true},
{"bafkrgqe3ohjcjplc6n4f3fwunlj6upltg.gn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4f.fyyxnayrtdi5oc4xb2332g645433aeg.ipfs.dweb.link", "dweb.link", "ipfs", "TODO", true},
*/
// other namespaces
{"api.localhost", "", "", "", false},
{"peerid.p2p.localhost", "localhost", "p2p", "peerid", true},
Expand Down
40 changes: 39 additions & 1 deletion test/sharness/t0114-gateway-subdomains.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ test_launch_ipfs_daemon --offline
test_expect_success "Add test text file" '
CID_VAL="hello"
CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q)
CIDv1_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q)
CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q)
CIDv0to1=$(echo "$CIDv0" | ipfs cid base32)
'
Expand Down Expand Up @@ -119,7 +120,6 @@ test_expect_success "Publish test text file to IPNS" '
test_cmp expected2 output
'


# ensure we start with empty Gateway.PublicGateways
test_expect_success 'start daemon with empty config for Gateway.PublicGateways' '
test_kill_ipfs_daemon &&
Expand Down Expand Up @@ -262,6 +262,7 @@ test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/
test_should_contain "subdir2-bar" list_response
'


# *.ipns.localhost

# <libp2p-key>.ipns.localhost
Expand Down Expand Up @@ -501,6 +502,43 @@ test_hostname_gateway_response_should_contain \
"http://127.0.0.1:$GWAY_PORT" \
"404 Not Found"

## ============================================================================
## Special handling of CIDs that do not fit in a single DNS Label (>63chars)
## https://github.com/ipfs/go-ipfs/issues/7318
## ============================================================================

CID_DNS_TODO="TODO"

# local: *.localhost
test_localhost_gateway_response_should_contain \
"request for a long CID at localhost/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \
"http://localhost:$GWAY_PORT/ipfs/$CIDv1_LONG" \
"Location: http://${CID_DNS_TODO}.ipfs.localhost:$GWAY_PORT/"

test_localhost_gateway_response_should_contain \
"request for {dns-fixed-long-CID}.ipfs.localhost should return expected payload" \
"http://${CID_DNS_TODO}.ipfs.localhost:$GWAY_PORT" \
"$CID_VAL"

# public gateway: *.example.com

test_hostname_gateway_response_should_contain \
"request for a long CID at example.com/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \
"example.com" \
"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_LONG" \
"Location: http://${CID_DNS_TODO}.ipfs.example.com"

test_hostname_gateway_response_should_contain \
"request for {long.CID}.ipfs.example.com should return expected payload" \
"${CID_DNS_TODO}.ipfs.example.com" \
"http://127.0.0.1:$GWAY_PORT" \
"$CID_VAL"

test_hostname_gateway_response_should_contain \
"request for {dns-fixed-long-CID}.ipfs.example.com should return redirect to a canonical Origin" \
"${CID_DNS_TODO}.ipfs.example.com" \
"http://127.0.0.1:$GWAY_PORT" \
"Location: http://${CID_DNS_TODO}.ipfs.example.com"

## ============================================================================
## Test path-based requests with a custom hostname config
Expand Down

0 comments on commit cebf519

Please sign in to comment.