Skip to content
This repository has been archived by the owner on Jan 8, 2024. It is now read-only.

feat!: support DNS over HTTPS and DNS-JSON over HTTPS #55

Merged
merged 16 commits into from
Dec 5, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion packages/ipns/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@
"./routing": {
"types": "./dist/src/routing/index.d.ts",
"import": "./dist/src/routing/index.js"
},
"./dns-resolvers": {
"types": "./dist/src/dns-resolvers/index.d.ts",
"import": "./dist/src/dns-resolvers/index.js"
}
},
"eslintConfig": {
Expand Down Expand Up @@ -166,6 +170,7 @@
"@libp2p/logger": "^2.0.6",
"@libp2p/peer-id": "^2.0.1",
"@libp2p/record": "^3.0.0",
"dns-packet": "^5.6.0",
"hashlru": "^2.3.0",
"interface-datastore": "^8.0.0",
"ipns": "^6.0.0",
Expand All @@ -183,7 +188,7 @@
"sinon-ts": "^1.0.0"
},
"browser": {
"./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js"
"./dist/src/dns-resolvers/default.js": "./dist/src/dns-resolvers/default.browser.js"
},
"typedoc": {
"entryPoint": "./src/index.ts"
Expand Down
77 changes: 77 additions & 0 deletions packages/ipns/src/dns-resolvers/default.browser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/* eslint-env browser */

import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import { TLRU } from '../utils/tlru.js'
import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'
import type { DNSResponse } from '../utils/dns.js'

// Avoid sending multiple queries for the same hostname by caching results
const cache = new TLRU<string>(1000)
// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884
// However we know browsers themselves cache DNS records for at least 1 minute,
// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426
const ttl = 60 * 1000

// browsers limit concurrent connections per host,
// we don't want to exhaust the limit (~6)
const httpQueue = new PQueue({ concurrency: 4 })

const ipfsPath = (response: { Path: string, Message: string }): string => {
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
if (response.Path != null) {
return response.Path
}

throw new Error(response.Message)
}

export function defaultResolver (): DNSResolver {
return async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const searchParams = new URLSearchParams()
searchParams.set('arg', fqdn)

const query = searchParams.toString()

// try cache first
if (options.nocache !== true && cache.has(query)) {
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
const response = cache.get(query)

if (response != null) {
options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
return response
}
}

options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))

// fallback to delegated DNS resolver
const response = await httpQueue.add(async () => {
// Delegated HTTP resolver sending DNSLink queries to ipfs.io
const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`, {
signal: options.signal
})
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
const query = new URL(res.url).search.slice(1)
const json = await res.json()

options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))

const response = ipfsPath(json)

cache.set(query, response, ttl)

return response
}, {
signal: options.signal
})

if (response == null) {
throw new Error('No DNS response received')
}

return response
}

return resolve(fqdn, options)
}
}
39 changes: 39 additions & 0 deletions packages/ipns/src/dns-resolvers/default.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { Resolver } from 'node:dns/promises'
import { CodeError } from '@libp2p/interfaces/errors'
import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'
import type { AbortOptions } from '@libp2p/interfaces'

export function defaultResolver (): DNSResolver {
return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}

async function resolve (domain: string, options: AbortOptions = {}): Promise<string> {
const resolver = new Resolver()
const listener = (): void => {
resolver.cancel()
}

options.signal?.addEventListener('abort', listener)

try {
const DNSLINK_REGEX = /^dnslink=.+$/
const records = await resolver.resolveTxt(domain)
const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), [])
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
.filter(record => DNSLINK_REGEX.test(record))

const dnslinkRecord = dnslinkRecords[0]
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved

// we now have dns text entries as an array of strings
// only records passing the DNSLINK_REGEX text are included
if (dnslinkRecord == null) {
throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND')
}

return dnslinkRecord
} finally {
options.signal?.removeEventListener('abort', listener)
}
}
90 changes: 90 additions & 0 deletions packages/ipns/src/dns-resolvers/dns-json-over-https.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* eslint-env browser */

import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import { type DNSResponse, findTTL, ipfsPath, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
import { TLRU } from '../utils/tlru.js'
import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'

// Avoid sending multiple queries for the same hostname by caching results
const cache = new TLRU<string>(1000)
// This TTL will be used if the remote service does not return one
const ttl = 60 * 1000

/**
* Uses the RFC 8427 'application/dns-json' content-type to resolve DNS queries.
*
* Supports and server that uses the same schema as Google's DNS over HTTPS
* resolver.
*
* This resolver needs fewer dependencies than the regular DNS-over-HTTPS
* resolver so can result in a smaller bundle size and consequently is preferred
* for browser use.
*
* @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/
* @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
* @see https://dnsprivacy.org/public_resolvers/
* @see https://datatracker.ietf.org/doc/html/rfc8427
*/
export function dnsJsonOverHttps (url: string): DNSResolver {
// browsers limit concurrent connections per host,
// we don't want preload calls to exhaust the limit (~6)
const httpQueue = new PQueue({ concurrency: 4 })

const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const searchParams = new URLSearchParams()
searchParams.set('name', fqdn)
searchParams.set('type', 'TXT')

const query = searchParams.toString()

// try cache first
if (options.nocache !== true && cache.has(query)) {
const response = cache.get(query)

if (response != null) {
options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
return response
}
}
Comment on lines +42 to +49

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this can be DRYed.

Copy link
Member

@lidel lidel Jul 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fwiw there is dns-over-http-resolver which is already used by js-mutiaddr which is already pulled in (i would assume) by Helia users.
Perhaps DoH code should be improved there, so we reuse it for all DNS lookup needs, and here we only import it?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added the logic here: vasco-santos/dns-over-http-resolver#102

 * This resolver needs fewer dependencies than the regular DNS-over-HTTPS
 * resolver so can result in a smaller bundle size and consequently is preferred
 * for browser use.

FWIW, I am not sure how much of a performance hit this is, I think having one good way of doing this should be acceptable, dns-over-https-resolver can prolly be refactored further to export a lighter version ESM for browser use.


options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))

// query DNS-JSON over HTTPS server
const response = await httpQueue.add(async () => {
const res = await fetch(`${url}?${searchParams}`, {
headers: {
accept: 'application/dns-json'
},
signal: options.signal
})

if (res.status !== 200) {
throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)
}

const query = new URL(res.url).search.slice(1)
const json: DNSResponse = await res.json()

options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))

const result = ipfsPath(fqdn, json)

cache.set(query, result, findTTL(fqdn, json) ?? ttl)

return result
}, {
signal: options.signal
})

if (response == null) {
throw new Error('No DNS response received')
}

return response
}

return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}
143 changes: 143 additions & 0 deletions packages/ipns/src/dns-resolvers/dns-over-https.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/* eslint-env browser */

import { Buffer } from 'buffer'
import dnsPacket from 'dns-packet'
import { base64url } from 'multiformats/bases/base64'
import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { type DNSResponse, findTTL, ipfsPath, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
import { TLRU } from '../utils/tlru.js'
import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'

// Avoid sending multiple queries for the same hostname by caching results
const cache = new TLRU<string>(1000)
// This TTL will be used if the remote service does not return one
const ttl = 60 * 1000

/**
* Uses the RFC 1035 'application/dns-message' content-type to resolve DNS
* queries.
*
* This resolver needs more dependencies than the non-standard
* DNS-JSON-over-HTTPS resolver so can result in a larger bundle size and
* consequently is not preferred for browser use.
*
* @see https://datatracker.ietf.org/doc/html/rfc1035
* @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-wireformat/
* @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
* @see https://dnsprivacy.org/public_resolvers/
*/
export function dnsOverHttps (url: string): DNSResolver {
// browsers limit concurrent connections per host,
// we don't want preload calls to exhaust the limit (~6)
const httpQueue = new PQueue({ concurrency: 4 })

const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const dnsQuery = dnsPacket.encode({
type: 'query',
id: 0,
flags: dnsPacket.RECURSION_DESIRED,
questions: [{
type: 'TXT',
name: fqdn
}]
})

const searchParams = new URLSearchParams()
searchParams.set('dns', base64url.encode(dnsQuery).substring(1))

const query = searchParams.toString()

// try cache first
if (options.nocache !== true && cache.has(query)) {
const response = cache.get(query)

if (response != null) {
options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
return response
}
}
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved

options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))

// query DNS over HTTPS server
const response = await httpQueue.add(async () => {
const res = await fetch(`${url}?${searchParams}`, {
headers: {
accept: 'application/dns-message'
},
signal: options.signal
})

if (res.status !== 200) {
throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)
}

const query = new URL(res.url).search.slice(1)
const buf = await res.arrayBuffer()
// map to expected response format
const json = toDNSResponse(dnsPacket.decode(Buffer.from(buf)))

options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))

const result = ipfsPath(fqdn, json)
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved

cache.set(query, result, findTTL(fqdn, json) ?? ttl)

return json
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
}, {
signal: options.signal
})

if (response == null) {
throw new Error('No DNS response received')
}

return ipfsPath(fqdn, response)
}

return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}

function toDNSResponse (response: dnsPacket.Packet): DNSResponse {
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
const txtType = 16

return {
Status: 0,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need the status field? If so, what for? it's not part of rfc8427 nor rfc1035

// @ts-expect-error field is missing from types
TC: Boolean(response.flag_tc) || false,
// @ts-expect-error field is missing from types
RD: Boolean(response.flag_rd) || false,
// @ts-expect-error field is missing from types
RA: Boolean(response.flag_ra) || false,
// @ts-expect-error field is missing from types
AD: Boolean(response.flag_ad) || false,
// @ts-expect-error field is missing from types
CD: Boolean(response.flag_cd) || false,
Question: response.questions?.map(q => ({
name: q.name,
type: txtType
})) ?? [],
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
Answer: response.answers?.map(a => {
if (a.type !== 'TXT' || a.data.length < 1) {
return {
name: a.name,
type: txtType,
TTL: 0,
data: 'invalid'
}
}

return {
name: a.name,
type: txtType,
TTL: a.ttl ?? ttl,
// @ts-expect-error we have already checked that a.data is not empty
data: uint8ArrayToString(a.data[0])
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
}
}) ?? []
}
}
3 changes: 3 additions & 0 deletions packages/ipns/src/dns-resolvers/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

export { dnsOverHttps } from './dns-over-https.js'
export { dnsJsonOverHttps } from './dns-json-over-https.js'
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
Loading