Skip to content
This repository has been archived by the owner on Jan 8, 2024. It is now read-only.

feat!: support DNS over HTTPS and DNS-JSON over HTTPS #55

Merged
merged 16 commits into from
Dec 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@ node_modules
package-lock.json
yarn.lock
.vscode
.env
.envrc
.tool-versions
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop"
},
"devDependencies": {
"aegir": "^41.0.0",
"aegir": "^41.1.14",
"npm-run-all": "^4.1.5"
},
"type": "module",
Expand Down
2 changes: 1 addition & 1 deletion packages/interop/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default {
host: '127.0.0.1',
port: ipfsdPort
}, {
ipfsBin: (await import('go-ipfs')).default.path(),
ipfsBin: (await import('kubo')).default.path(),
kuboRpcModule: kuboRpcClient,
ipfsOptions: {
config: {
Expand Down
5 changes: 2 additions & 3 deletions packages/interop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,15 @@
"@libp2p/peer-id-factory": "^3.0.3",
"@libp2p/tcp": "^8.0.4",
"@libp2p/websockets": "^7.0.4",
"aegir": "^41.0.0",
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be left in place. Each package in a monorepo should declare all of it's dependencies, otherwise we have some deps here, some deps there, and then it becomes hard to reason about where things are coming from and why.

"blockstore-core": "^4.0.1",
"datastore-core": "^9.0.3",
"go-ipfs": "^0.22.0",
"helia": "^2.0.1",
"ipfsd-ctl": "^13.0.0",
"ipns": "^7.0.1",
"it-all": "^3.0.2",
"it-last": "^3.0.1",
"it-map": "^3.0.3",
"kubo": "^0.24.0",
"kubo-rpc-client": "^3.0.0",
"libp2p": "^0.46.6",
"merge-options": "^3.0.4",
Expand All @@ -79,7 +78,7 @@
},
"browser": {
"./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js",
"go-ipfs": false
"kubo": false
},
"private": true
}
8 changes: 5 additions & 3 deletions packages/interop/test/dht.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,11 @@ keyTypes.forEach(type => {
message: 'Kubo could not find Helia on the DHT'
})

name = ipns(helia, [
dht(helia)
])
name = ipns(helia, {
routers: [
dht(helia)
]
})
}

afterEach(async () => {
Expand Down
6 changes: 2 additions & 4 deletions packages/interop/test/fixtures/create-kubo.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped
import * as goIpfs from 'go-ipfs'
import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl'
import * as kubo from 'kubo'
import * as kuboRpcClient from 'kubo-rpc-client'
import mergeOptions from 'merge-options'
import { isElectronMain, isNode } from 'wherearewe'

export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise<Controller> {
const opts = mergeOptions({
kuboRpcModule: kuboRpcClient,
ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined,
ipfsBin: isNode || isElectronMain ? kubo.path() : undefined,
test: true,
endpoint: process.env.IPFSD_SERVER,
ipfsOptions: {
Expand Down
8 changes: 5 additions & 3 deletions packages/interop/test/pubsub.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,11 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => {
// connect the two nodes
await connect(helia, kubo, '/meshsub/1.1.0')

name = ipns(helia, [
pubsub(helia)
])
name = ipns(helia, {
routers: [
pubsub(helia)
]
})
})

afterEach(async () => {
Expand Down
15 changes: 12 additions & 3 deletions packages/ipns/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
"./routing": {
"types": "./dist/src/routing/index.d.ts",
"import": "./dist/src/routing/index.js"
},
"./dns-resolvers": {
"types": "./dist/src/dns-resolvers/index.d.ts",
"import": "./dist/src/dns-resolvers/index.js"
}
},
"eslintConfig": {
Expand Down Expand Up @@ -155,10 +159,11 @@
"release": "aegir release"
},
"dependencies": {
"@libp2p/interface": "^0.1.2",
"@libp2p/kad-dht": "^10.0.11",
"@libp2p/logger": "^3.0.2",
"@libp2p/peer-id": "^3.0.2",
"dns-over-http-resolver": "^2.1.3",
"dns-packet": "^5.6.0",
"hashlru": "^2.3.0",
"interface-datastore": "^8.0.0",
"ipns": "^7.0.1",
Expand All @@ -169,13 +174,17 @@
"uint8arrays": "^4.0.3"
},
"devDependencies": {
"@libp2p/interface": "^0.1.4",
"@libp2p/peer-id-factory": "^3.0.3",
"aegir": "^41.0.0",
"@types/dns-packet": "^5.6.4",
"datastore-core": "^9.0.3",
"sinon": "^17.0.0",
"sinon-ts": "^1.0.0"
},
"browser": {
"./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js"
"./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js"
},
"typedoc": {
"entryPoint": "./src/index.ts"
}
}
9 changes: 9 additions & 0 deletions packages/ipns/src/dns-resolvers/default.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
import resolve from './resolver.js'
import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'

export function defaultResolver (): DNSResolver {
return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}
90 changes: 90 additions & 0 deletions packages/ipns/src/dns-resolvers/dns-json-over-https.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* eslint-env browser */

import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js'
import { TLRU } from '../utils/tlru.js'
import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'

// Avoid sending multiple queries for the same hostname by caching results
const cache = new TLRU<string>(1000)
// This TTL will be used if the remote service does not return one
const ttl = 60 * 1000

/**
* Uses the RFC 8427 'application/dns-json' content-type to resolve DNS queries.
*
* Supports and server that uses the same schema as Google's DNS over HTTPS
* resolver.
*
* This resolver needs fewer dependencies than the regular DNS-over-HTTPS
* resolver so can result in a smaller bundle size and consequently is preferred
* for browser use.
*
* @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/
* @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
* @see https://dnsprivacy.org/public_resolvers/
* @see https://datatracker.ietf.org/doc/html/rfc8427
*/
export function dnsJsonOverHttps (url: string): DNSResolver {
// browsers limit concurrent connections per host,
// we don't want preload calls to exhaust the limit (~6)
const httpQueue = new PQueue({ concurrency: 4 })

const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const searchParams = new URLSearchParams()
searchParams.set('name', fqdn)
searchParams.set('type', 'TXT')

const query = searchParams.toString()

// try cache first
if (options.nocache !== true && cache.has(query)) {
const response = cache.get(query)

if (response != null) {
options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
return response
}
}
Comment on lines +42 to +49

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this can be DRYed.

Copy link
Member

@lidel lidel Jul 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fwiw there is dns-over-http-resolver which is already used by js-mutiaddr which is already pulled in (i would assume) by Helia users.
Perhaps DoH code should be improved there, so we reuse it for all DNS lookup needs, and here we only import it?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added the logic here: vasco-santos/dns-over-http-resolver#102

 * This resolver needs fewer dependencies than the regular DNS-over-HTTPS
 * resolver so can result in a smaller bundle size and consequently is preferred
 * for browser use.

FWIW, I am not sure how much of a performance hit this is, I think having one good way of doing this should be acceptable, dns-over-https-resolver can prolly be refactored further to export a lighter version ESM for browser use.


options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))

// query DNS-JSON over HTTPS server
const response = await httpQueue.add(async () => {
const res = await fetch(`${url}?${searchParams}`, {
headers: {
accept: 'application/dns-json'
},
signal: options.signal
})

if (res.status !== 200) {
throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)

Check warning on line 63 in packages/ipns/src/dns-resolvers/dns-json-over-https.ts

View check run for this annotation

Codecov / codecov/patch

packages/ipns/src/dns-resolvers/dns-json-over-https.ts#L63

Added line #L63 was not covered by tests
}

const query = new URL(res.url).search.slice(1)
const json: DNSResponse = await res.json()

options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))

const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json)

cache.set(query, ipfsPath, answer.TTL ?? ttl)

return ipfsPath
}, {
signal: options.signal
})

if (response == null) {
throw new Error('No DNS response received')

Check warning on line 81 in packages/ipns/src/dns-resolvers/dns-json-over-https.ts

View check run for this annotation

Codecov / codecov/patch

packages/ipns/src/dns-resolvers/dns-json-over-https.ts#L81

Added line #L81 was not covered by tests
}

return response
}

return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}
146 changes: 146 additions & 0 deletions packages/ipns/src/dns-resolvers/dns-over-https.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/* eslint-env browser */

import { Buffer } from 'buffer'
import dnsPacket, { type DecodedPacket } from 'dns-packet'
import { base64url } from 'multiformats/bases/base64'
import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js'
import { TLRU } from '../utils/tlru.js'
import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'

// Avoid sending multiple queries for the same hostname by caching results
const cache = new TLRU<string>(1000)
// This TTL will be used if the remote service does not return one
const ttl = 60 * 1000

/**
* Uses the RFC 1035 'application/dns-message' content-type to resolve DNS
* queries.
*
* This resolver needs more dependencies than the non-standard
* DNS-JSON-over-HTTPS resolver so can result in a larger bundle size and
* consequently is not preferred for browser use.
*
* @see https://datatracker.ietf.org/doc/html/rfc1035
* @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-wireformat/
* @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
* @see https://dnsprivacy.org/public_resolvers/
*/
export function dnsOverHttps (url: string): DNSResolver {
// browsers limit concurrent connections per host,
// we don't want preload calls to exhaust the limit (~6)
const httpQueue = new PQueue({ concurrency: 4 })

const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
const dnsQuery = dnsPacket.encode({
type: 'query',
id: 0,
flags: dnsPacket.RECURSION_DESIRED,
questions: [{
type: 'TXT',
name: fqdn
}]
})

const searchParams = new URLSearchParams()
searchParams.set('dns', base64url.encode(dnsQuery).substring(1))

const query = searchParams.toString()

// try cache first
if (options.nocache !== true && cache.has(query)) {
const response = cache.get(query)

if (response != null) {
options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
return response
}
}
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved

options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))

// query DNS over HTTPS server
const response = await httpQueue.add(async () => {
const res = await fetch(`${url}?${searchParams}`, {
headers: {
accept: 'application/dns-message'
},
signal: options.signal
})

if (res.status !== 200) {
throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)

Check warning on line 74 in packages/ipns/src/dns-resolvers/dns-over-https.ts

View check run for this annotation

Codecov / codecov/patch

packages/ipns/src/dns-resolvers/dns-over-https.ts#L74

Added line #L74 was not covered by tests
}

const query = new URL(res.url).search.slice(1)
const buf = await res.arrayBuffer()
// map to expected response format
const json = toDNSResponse(dnsPacket.decode(Buffer.from(buf)))

options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))

const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json)

cache.set(query, ipfsPath, answer.TTL ?? ttl)

return ipfsPath
}, {
signal: options.signal
})

if (response == null) {
throw new Error('No DNS response received')

Check warning on line 94 in packages/ipns/src/dns-resolvers/dns-over-https.ts

View check run for this annotation

Codecov / codecov/patch

packages/ipns/src/dns-resolvers/dns-over-https.ts#L94

Added line #L94 was not covered by tests
}

return response
}

return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
}
}

function toDNSResponse (response: DecodedPacket): DNSResponse {
const txtType = 16

return {
Status: 0,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need the status field? If so, what for? it's not part of rfc8427 nor rfc1035

TC: response.flag_tc ?? false,
RD: response.flag_rd ?? false,
RA: response.flag_ra ?? false,
AD: response.flag_ad ?? false,
CD: response.flag_cd ?? false,
Question: response.questions?.map(q => ({
name: q.name,
type: txtType
})) ?? [],
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
Answer: response.answers?.map(a => {
if (a.type !== 'TXT' || a.data.length < 1) {
return {
name: a.name,
type: txtType,
TTL: 0,
data: 'invalid'
}
}

if (!Buffer.isBuffer(a.data[0])) {
return {
name: a.name,
type: txtType,
TTL: a.ttl ?? ttl,
data: String(a.data[0])
}

Check warning on line 135 in packages/ipns/src/dns-resolvers/dns-over-https.ts

View check run for this annotation

Codecov / codecov/patch

packages/ipns/src/dns-resolvers/dns-over-https.ts#L130-L135

Added lines #L130 - L135 were not covered by tests
}

return {
name: a.name,
type: txtType,
TTL: a.ttl ?? ttl,
data: uint8ArrayToString(a.data[0])
}
}) ?? []
}
}
2 changes: 2 additions & 0 deletions packages/ipns/src/dns-resolvers/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export { dnsOverHttps } from './dns-over-https.js'
export { dnsJsonOverHttps } from './dns-json-over-https.js'
SgtPooki marked this conversation as resolved.
Show resolved Hide resolved
Loading