Add cache option

index.js

@@ -16,6 +16,7 @@ const unzipResponse = require('unzip-response');
 const createErrorClass = require('create-error-class');
 const isRetryAllowed = require('is-retry-allowed');
 const Buffer = require('safe-buffer').Buffer;
+const cacheableRequest = require('cacheable-request');
 const pkg = require('./package');
 
 function requestAsEventEmitter(opts) {
@@ -30,7 +31,7 @@ function requestAsEventEmitter(opts) {
 	const get = opts => {
 		const fn = opts.protocol === 'https:' ? https : http;
 
-		const req = fn.request(opts, res => {
+		const cacheReq = cacheableRequest(fn.request, opts, res => {
 			const statusCode = res.statusCode;
 
 			if (isRedirect(statusCode) && opts.followRedirect && 'location' in res.headers && (opts.method === 'GET' || opts.method === 'HEAD')) {
@@ -54,30 +55,30 @@ function requestAsEventEmitter(opts) {
 			}
 
 			setImmediate(() => {
-				const response = typeof unzipResponse === 'function' && req.method !== 'HEAD' ? unzipResponse(res) : res;
+				const response = typeof unzipResponse === 'function' && opts.method !== 'HEAD' ? unzipResponse(res) : res;
 				response.url = redirectUrl || requestUrl;
 				response.requestUrl = requestUrl;
 
 				ee.emit('response', response);
 			});
 		});
 
-		req.once('error', err => {
-			const backoff = opts.retries(++retryCount, err);
+		cacheReq.on('request', req => {
+			req.once('error', err => {
+				const backoff = opts.retries(++retryCount, err);
 
-			if (backoff) {
-				setTimeout(get, backoff, opts);
-				return;
-			}
+				if (backoff) {
+					setTimeout(get, backoff, opts);
+					return;
+				}
 
-			ee.emit('error', new got.RequestError(err, opts));
-		});
+				ee.emit('error', new got.RequestError(err, opts));
+			});
 
-		if (opts.gotTimeout) {
-			timedOut(req, opts.gotTimeout);
-		}
+			if (opts.gotTimeout) {
+				timedOut(req, opts.gotTimeout);
+			}
 
-		setImmediate(() => {
 			ee.emit('request', req);
 		});
 	};

package.json

@@ -42,6 +42,7 @@
     "fetch"
   ],
   "dependencies": {
+    "cacheable-request": "^0.3.0",
     "create-error-class": "^3.0.0",
     "duplexer3": "^0.1.4",
     "get-stream": "^3.0.0",
@@ -59,6 +60,7 @@
     "coveralls": "^2.11.4",
     "form-data": "^2.1.1",
     "get-port": "^3.0.0",
+    "get-stream": "^3.0.0",
     "into-stream": "^3.0.0",
     "nyc": "^10.0.0",
     "pem": "^1.4.4",

test/cache.js

@@ -0,0 +1,197 @@
+import test from 'ava';
+import getStream from 'get-stream';
+import got from '../';
+import {createServer} from './helpers/server';
+
+let s;
+
+test.before('setup', async () => {
+	s = await createServer();
+
+	let noStoreIndex = 0;
+	s.on('/no-store', (req, res) => {
+		noStoreIndex++;
+		res.setHeader('Cache-Control', 'public, no-cache, no-store');
+		res.end(noStoreIndex.toString());
+	});
+
+	let cacheIndex = 0;
+	s.on('/cache', (req, res) => {
+		cacheIndex++;
+		res.setHeader('Cache-Control', 'public, max-age=60');
+		res.end(cacheIndex.toString());
+	});
+
+	s.on('/last-modified', (req, res) => {
+		res.setHeader('Cache-Control', 'public, max-age=0');
+		res.setHeader('Last-Modified', 'Wed, 21 Oct 2015 07:28:00 GMT');
+		let responseBody = 'last-modified';
+
+		if (req.headers['if-modified-since'] === 'Wed, 21 Oct 2015 07:28:00 GMT') {
+			res.statusCode = 304;
+			responseBody = null;
+		}
+
+		res.end(responseBody);
+	});
+
+	s.on('/etag', (req, res) => {
+		res.setHeader('Cache-Control', 'public, max-age=0');
+		res.setHeader('ETag', '33a64df551425fcc55e4d42a148795d9f25f89d4');
+		let responseBody = 'etag';
+
+		if (req.headers['if-none-match'] === '33a64df551425fcc55e4d42a148795d9f25f89d4') {
+			res.statusCode = 304;
+			responseBody = null;
+		}
+
+		res.end(responseBody);
+	});
+
+	let cacheThenNoStoreIndex = 0;
+	s.on('/cache-then-no-store-on-revalidate', (req, res) => {
+		const cc = cacheThenNoStoreIndex === 0 ? 'public, max-age=0' : 'public, no-cache, no-store';
+		cacheThenNoStoreIndex++;
+		res.setHeader('Cache-Control', cc);
+		res.end('cache-then-no-store-on-revalidate');
+	});
+
+	await s.listen(s.port);
+});
+
+test('Non cacheable responses are not cached', async t => {
+	const endpoint = '/no-store';
+	const cache = new Map();
+
+	const firstResponseInt = parseInt((await got(s.url + endpoint, {cache})).body, 10);
+	const secondResponseInt = parseInt((await got(s.url + endpoint, {cache})).body, 10);
+
+	t.is(cache.size, 0);
+	t.true(firstResponseInt < secondResponseInt);
+});
+
+test('Cacheable responses are cached', async t => {
+	const endpoint = '/cache';
+	const cache = new Map();
+
+	const firstResponse = await got(s.url + endpoint, {cache});
+	const secondResponse = await got(s.url + endpoint, {cache});
+
+	t.is(cache.size, 1);
+	t.is(firstResponse.body, secondResponse.body);
+});
+
+test('Stream responses are cached', async t => {
+	const endpoint = '/cache';
+	const cache = new Map();
+
+	const firstResponseBody = await getStream(got.stream(s.url + endpoint, {cache}));
+	const secondResponseBody = await getStream(got.stream(s.url + endpoint, {cache}));
+
+	t.is(cache.size, 1);
+	t.is(firstResponseBody, secondResponseBody);
+});
+
+test('Binary responses are cached', async t => {
+	const endpoint = '/cache';
+	const cache = new Map();
+	const encoding = null;
+
+	const firstResponse = await got(s.url + endpoint, {cache, encoding});
+	const secondResponse = await got(s.url + endpoint, {cache, encoding});
+
+	t.is(cache.size, 1);
+	t.true(firstResponse.body instanceof Buffer);
+	t.is(firstResponse.body.toString(), secondResponse.body.toString());
+});
+
+test('TTL is passed to cache', async t => {
+	const endpoint = '/cache';
+	const store = new Map();
+	const cache = {
+		get: store.get.bind(store),
+		set: (key, val, ttl) => {
+			t.true(typeof ttl === 'number');
+			t.true(ttl > 0);
+			return store.set(key, val, ttl);
+		},
+		delete: store.delete.bind(store)
+	};
+
+	t.plan(2);
+
+	await got(s.url + endpoint, {cache});
+});
+
+test('Cached response is re-encoded to current encoding option', async t => {
+	const endpoint = '/cache';
+	const cache = new Map();
+	const firstEncoding = 'base64';
+	const secondEncoding = 'hex';
+
+	const firstResponse = await got(s.url + endpoint, {cache, encoding: firstEncoding});
+	const secondResponse = await got(s.url + endpoint, {cache, encoding: secondEncoding});
+
+	const expectedSecondResponseBody = Buffer.from(firstResponse.body, firstEncoding).toString(secondEncoding);
+
+	t.is(cache.size, 1);
+	t.is(secondResponse.body, expectedSecondResponseBody);
+});
+
+test('Stale cache entries with Last-Modified headers are revalidated', async t => {
+	const endpoint = '/last-modified';
+	const cache = new Map();
+
+	const firstResponse = await got(s.url + endpoint, {cache});
+	const secondResponse = await got(s.url + endpoint, {cache});
+
+	t.is(cache.size, 1);
+	t.is(firstResponse.statusCode, 200);
+	t.is(secondResponse.statusCode, 304);
+	t.is(firstResponse.body, 'last-modified');
+	t.is(firstResponse.body, secondResponse.body);
+});
+
+test('Stale cache entries with ETag headers are revalidated', async t => {
+	const endpoint = '/etag';
+	const cache = new Map();
+
+	const firstResponse = await got(s.url + endpoint, {cache});
+	const secondResponse = await got(s.url + endpoint, {cache});
+
+	t.is(cache.size, 1);
+	t.is(firstResponse.statusCode, 200);
+	t.is(secondResponse.statusCode, 304);
+	t.is(firstResponse.body, 'etag');
+	t.is(firstResponse.body, secondResponse.body);
+});
+
+test('Stale cache entries that can\'t be revalidate are deleted from cache', async t => {
+	const endpoint = '/cache-then-no-store-on-revalidate';
+	const cache = new Map();
+
+	const firstResponse = await got(s.url + endpoint, {cache});
+	t.is(cache.size, 1);
+	const secondResponse = await got(s.url + endpoint, {cache, log: true});
+
+	t.is(cache.size, 0);
+	t.is(firstResponse.statusCode, 200);
+	t.is(secondResponse.statusCode, 200);
+	t.is(firstResponse.body, 'cache-then-no-store-on-revalidate');
+	t.is(firstResponse.body, secondResponse.body);
+});
+
+test('Response objects have fromCache property set correctly', async t => {
+	const endpoint = '/cache';
+	const cache = new Map();
+
+	const response = await got(s.url + endpoint, {cache});
+	const cachedResponse = await got(s.url + endpoint, {cache});
+
+	t.false(response.fromCache);
+	t.true(cachedResponse.fromCache);
+});
+
+test.after('cleanup', async () => {
+	await s.close();
+});