From a6177c5cef4d76361901f322dd7f804cc7a052cb Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Tue, 24 Sep 2024 11:23:57 +0530 Subject: [PATCH 01/11] Added ClearlyDefined as User-Agent in default header --- lib/fetch.js | 4 +++- test/unit/lib/fetchTests.js | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/fetch.js b/lib/fetch.js index 056152b8..0fd30130 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -4,6 +4,8 @@ const axios = require('axios') function buildRequestOptions(request) { + const defaultHeaders = { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } + let responseType = 'text' if (request.json) { responseType = 'json' @@ -20,7 +22,7 @@ function buildRequestOptions(request) { method: request.method, url: request.url || request.uri, responseType, - headers: request.headers, + headers: Object.assign(defaultHeaders, request.headers), data: request.body, ...validateOptions } diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 330b3899..3a286070 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -129,6 +129,8 @@ describe('CallFetch', () => { const json = await requests[0].body.getJson() expect(json).to.deep.equal({ test: 'test' }) expect(requests[0].headers).to.include({ 'x-crawler': 'secret' }) + //Check for the default header value + expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) }) describe('test simple', () => { From 5f68863dbd3a7133ed34ed650389e3372c254f57 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Thu, 26 Sep 2024 11:52:32 +0530 Subject: [PATCH 02/11] Updated default header in axios global config --- lib/fetch.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/fetch.js b/lib/fetch.js index 0fd30130..10f19889 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -3,9 +3,9 @@ const axios = require('axios') +const defaultHeaders = { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } +axios.defaults.headers = defaultHeaders function buildRequestOptions(request) { - const defaultHeaders = { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } - let responseType = 'text' if (request.json) { responseType = 'json' @@ -22,7 +22,7 @@ function buildRequestOptions(request) { method: request.method, url: request.url || request.uri, responseType, - headers: Object.assign(defaultHeaders, request.headers), + headers: request.headers, data: request.body, ...validateOptions } From be3b10973449014a68decb63c935d2b6943344d9 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Fri, 11 Oct 2024 08:52:25 +0530 Subject: [PATCH 03/11] Freezed the defaultHeader object and exported it to maintain consistency throughout --- lib/fetch.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/fetch.js b/lib/fetch.js index 10f19889..eb6062bf 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -3,8 +3,10 @@ const axios = require('axios') -const defaultHeaders = { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } +const defaultHeaders = Object.freeze({ 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + axios.defaults.headers = defaultHeaders + function buildRequestOptions(request) { let responseType = 'text' if (request.json) { @@ -47,4 +49,4 @@ function withDefaults(opts) { return request => callFetch(request, axiosInstance) } -module.exports = { callFetch, withDefaults } +module.exports = { callFetch, withDefaults, defaultHeaders } From 0dee0136670632056bd7b17e77c2534c51d7a0cc Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Fri, 11 Oct 2024 08:53:54 +0530 Subject: [PATCH 04/11] Removed user-agent header from requests explicitly stated --- providers/fetch/cratesioFetch.js | 4 +--- providers/fetch/mavenBasedFetch.js | 6 ++---- providers/fetch/packagistFetch.js | 5 +---- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/providers/fetch/cratesioFetch.js b/providers/fetch/cratesioFetch.js index 9d9a8e40..b6706db9 100644 --- a/providers/fetch/cratesioFetch.js +++ b/providers/fetch/cratesioFetch.js @@ -51,8 +51,7 @@ class CratesioFetch extends AbstractFetch { try { registryData = await request({ url: `https://crates.io/api/v1/crates/${spec.name}`, - json: true, - headers: { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } + json: true }) } catch (exception) { if (exception.statusCode !== 404) throw exception @@ -72,7 +71,6 @@ class CratesioFetch extends AbstractFetch { url: `https://crates.io${version.dl_path}`, encoding: null, headers: { - 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)', Accept: 'text/html' } }) diff --git a/providers/fetch/mavenBasedFetch.js b/providers/fetch/mavenBasedFetch.js index 36500455..11a5ea90 100644 --- a/providers/fetch/mavenBasedFetch.js +++ b/providers/fetch/mavenBasedFetch.js @@ -2,7 +2,7 @@ // SPDX-License-Identifier: MIT const AbstractFetch = require('./abstractFetch') -const { withDefaults } = require('../../lib/fetch') +const { callFetch, defaultHeaders } = require('../../lib/fetch') const nodeRequest = require('request') const { clone, get } = require('lodash') const { promisify } = require('util') @@ -23,13 +23,11 @@ const extensionMap = { jar: '.jar' } -const defaultHeaders = { headers: { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } } - class MavenBasedFetch extends AbstractFetch { constructor(providerMap, options) { super(options) this._providerMap = { ...providerMap } - this._handleRequestPromise = options.requestPromise || withDefaults(defaultHeaders) + this._handleRequestPromise = options.requestPromise || callFetch this._handleRequestStream = options.requestStream || nodeRequest.defaults(defaultHeaders).get } diff --git a/providers/fetch/packagistFetch.js b/providers/fetch/packagistFetch.js index 8f8ce338..d6520f98 100644 --- a/providers/fetch/packagistFetch.js +++ b/providers/fetch/packagistFetch.js @@ -61,10 +61,7 @@ class PackagistFetch extends AbstractFetch { if (!distUrl) return request.markSkip('Missing dist.url ') return new Promise((resolve, reject) => { const options = { - url: distUrl, - headers: { - 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' - } + url: distUrl } nodeRequest .get(options, (error, response) => { From 62b062658dc651e5ec942d931db9a3edf0fa3f10 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Fri, 11 Oct 2024 08:55:25 +0530 Subject: [PATCH 05/11] Updated testcases to check for user-agent header in requests --- test/unit/lib/fetchTests.js | 42 +++++++++++++++++++ .../providers/fetch/mavenBasedFetchTests.js | 24 +++++++---- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 3a286070..24556985 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -23,6 +23,48 @@ describe('CallFetch', () => { expect(response).to.be.deep.equal(JSON.parse(expected)) }) + it('checks if the default header user-agent and other header is present in crate components', async () => { + const path = '/crates.io/api/v1/crates/name/1.0.0/download' + const endpointMock = await mockServer.forGet(path).thenReply(200, 'success') + + await callFetch({ + url: mockServer.urlFor(path), + method: 'GET', + json: true, + encoding: null, + headers: { + Accept: 'text/html' + } + }) + const requests = await endpointMock.getSeenRequests() + expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + expect(requests[0].headers).to.include({ accept: 'text/html' }) + }) + + it('checks if the default header user-agent is present in crate components', async () => { + const path = '/crates.io/api/v1/crates/name' + const endpointMock = await mockServer.forGet(path).thenReply(200, 'success') + + await callFetch({ + url: mockServer.urlFor(path), + method: 'GET', + json: true + }) + const requests = await endpointMock.getSeenRequests() + expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + }) + + it('checks if the default header user-agent is present in packagist components', async () => { + const path = '/api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001' + const endpointMock = await mockServer.forGet(path).thenReply(200, 'success') + + await callFetch({ + url: mockServer.urlFor(path) + }) + const requests = await endpointMock.getSeenRequests() + expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + }) + it('checks if the full response is fetched', async () => { const path = '/registry.npmjs.com/redis/0.1.0' const expected = fs.readFileSync('test/fixtures/fetch/redis-0.1.0.json') diff --git a/test/unit/providers/fetch/mavenBasedFetchTests.js b/test/unit/providers/fetch/mavenBasedFetchTests.js index 5b3995cc..26844903 100644 --- a/test/unit/providers/fetch/mavenBasedFetchTests.js +++ b/test/unit/providers/fetch/mavenBasedFetchTests.js @@ -20,24 +20,34 @@ describe('MavenBasedFetch', () => { }) }) - describe('Integration test for component not found', function () { + describe('Integration test', function () { const path = '/remotecontent?filepath=' const mockServer = mockttp.getLocal() - beforeEach(async () => await mockServer.start()) - afterEach(async () => await mockServer.stop()) - - it('should handle maven components not found', async () => { - const handler = new MavenBasedFetch( + let endpointMock + let handler + beforeEach(async () => { + await mockServer.start() + handler = new MavenBasedFetch( { mavencentral: mockServer.urlFor(path) }, { logger: { log: sinon.stub() } } ) - await mockServer.forAnyRequest().thenReply(404) + endpointMock = await mockServer.forAnyRequest().thenReply(404) + }) + afterEach(async () => await mockServer.stop()) + + it('should handle maven components not found', async () => { const request = await handler.handle( new Request('test', 'cd:/maven/mavencentral/org.apache.httpcomponents/httpcore/4.') ) expect(request.processControl).to.be.equal('skip') }) + + it('should check for default header in any request', async () => { + await handler.handle(new Request('test', 'cd:/maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16')) + const requests = await endpointMock.getSeenRequests() + expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + }) }) }) From f27aaac9a1c8305cfd48f8b60616a46e2122504a Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Thu, 17 Oct 2024 10:22:34 +0530 Subject: [PATCH 06/11] Replace user-agent header with defaultHeader declared globally --- providers/fetch/mavenBasedFetch.js | 2 +- providers/fetch/packagistFetch.js | 4 +++- test/unit/lib/fetchTests.js | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/providers/fetch/mavenBasedFetch.js b/providers/fetch/mavenBasedFetch.js index 11a5ea90..88c59ced 100644 --- a/providers/fetch/mavenBasedFetch.js +++ b/providers/fetch/mavenBasedFetch.js @@ -28,7 +28,7 @@ class MavenBasedFetch extends AbstractFetch { super(options) this._providerMap = { ...providerMap } this._handleRequestPromise = options.requestPromise || callFetch - this._handleRequestStream = options.requestStream || nodeRequest.defaults(defaultHeaders).get + this._handleRequestStream = options.requestStream || nodeRequest.defaults({ headers: defaultHeaders }).get } canHandle(request) { diff --git a/providers/fetch/packagistFetch.js b/providers/fetch/packagistFetch.js index d6520f98..8ba76a8e 100644 --- a/providers/fetch/packagistFetch.js +++ b/providers/fetch/packagistFetch.js @@ -9,6 +9,7 @@ const nodeRequest = require('request') const { promisify } = require('util') const readdir = promisify(fs.readdir) const FetchResult = require('../../lib/fetchResult') +const { defaultHeaders } = require('../../lib/fetch') const providerMap = { packagist: 'https://repo.packagist.org/' @@ -61,7 +62,8 @@ class PackagistFetch extends AbstractFetch { if (!distUrl) return request.markSkip('Missing dist.url ') return new Promise((resolve, reject) => { const options = { - url: distUrl + url: distUrl, + headers: defaultHeaders } nodeRequest .get(options, (error, response) => { diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 24556985..70c9b9b4 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -1,5 +1,5 @@ const { fail } = require('assert') -const { callFetch, withDefaults } = require('../../../lib/fetch') +const { callFetch, withDefaults, defaultHeaders } = require('../../../lib/fetch') const { expect } = require('chai') const fs = require('fs') const mockttp = require('mockttp') @@ -37,7 +37,7 @@ describe('CallFetch', () => { } }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + expect(requests[0].headers).to.includes(defaultHeaders) expect(requests[0].headers).to.include({ accept: 'text/html' }) }) @@ -51,7 +51,7 @@ describe('CallFetch', () => { json: true }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + expect(requests[0].headers).to.include(defaultHeaders) }) it('checks if the default header user-agent is present in packagist components', async () => { @@ -62,7 +62,7 @@ describe('CallFetch', () => { url: mockServer.urlFor(path) }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + expect(requests[0].headers).to.include(defaultHeaders) }) it('checks if the full response is fetched', async () => { @@ -129,7 +129,7 @@ describe('CallFetch', () => { const url = mockServer.urlFor(path) const endpointMock = await mockServer.forGet(path).thenReply(200) - const defaultOptions = { headers: { 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } } + const defaultOptions = { headers: defaultHeaders } const requestWithDefaults = withDefaults(defaultOptions) await requestWithDefaults({ url }) await requestWithDefaults({ url }) @@ -172,7 +172,7 @@ describe('CallFetch', () => { expect(json).to.deep.equal({ test: 'test' }) expect(requests[0].headers).to.include({ 'x-crawler': 'secret' }) //Check for the default header value - expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + expect(requests[0].headers).to.include(defaultHeaders) }) describe('test simple', () => { From 62b2b457677e8484cacf465c21eca7bd93ada111 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Thu, 17 Oct 2024 10:59:56 +0530 Subject: [PATCH 07/11] Updated testcases with defaultHeader declared globally --- test/unit/lib/fetchTests.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 70c9b9b4..675bef58 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -4,6 +4,11 @@ const { expect } = require('chai') const fs = require('fs') const mockttp = require('mockttp') +function checkHeader(headers) { + for (const [key, value] of Object.entries(defaultHeaders)) { + expect(headers).to.have.property(key.toLowerCase()).that.equals(value); + } +} describe('CallFetch', () => { describe('with mock server', () => { const mockServer = mockttp.getLocal() @@ -37,7 +42,7 @@ describe('CallFetch', () => { } }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.includes(defaultHeaders) + checkHeader(requests[0].headers) expect(requests[0].headers).to.include({ accept: 'text/html' }) }) @@ -51,7 +56,7 @@ describe('CallFetch', () => { json: true }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include(defaultHeaders) + checkHeader(requests[0].headers) }) it('checks if the default header user-agent is present in packagist components', async () => { @@ -62,7 +67,7 @@ describe('CallFetch', () => { url: mockServer.urlFor(path) }) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include(defaultHeaders) + checkHeader(requests[0].headers) }) it('checks if the full response is fetched', async () => { @@ -137,9 +142,9 @@ describe('CallFetch', () => { const requests = await endpointMock.getSeenRequests() expect(requests.length).to.equal(2) expect(requests[0].url).to.equal(url) - expect(requests[0].headers).to.include(defaultOptions.headers) + checkHeader(requests[0].headers) expect(requests[1].url).to.equal(url) - expect(requests[1].headers).to.include(defaultOptions.headers) + checkHeader(requests[1].headers) }) it('checks if the response is text with uri option in GET request', async () => { @@ -172,7 +177,7 @@ describe('CallFetch', () => { expect(json).to.deep.equal({ test: 'test' }) expect(requests[0].headers).to.include({ 'x-crawler': 'secret' }) //Check for the default header value - expect(requests[0].headers).to.include(defaultHeaders) + checkHeader(requests[0].headers) }) describe('test simple', () => { From ae5e87dcb397fda845a40361f2e2c070ed803043 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Thu, 17 Oct 2024 11:16:20 +0530 Subject: [PATCH 08/11] Fix Formatting --- test/unit/lib/fetchTests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 675bef58..53019c63 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -6,8 +6,8 @@ const mockttp = require('mockttp') function checkHeader(headers) { for (const [key, value] of Object.entries(defaultHeaders)) { - expect(headers).to.have.property(key.toLowerCase()).that.equals(value); - } + expect(headers).to.have.property(key.toLowerCase()).that.equals(value) + } } describe('CallFetch', () => { describe('with mock server', () => { From 45fba512e849617a02282914dfe090f26765dd0b Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Thu, 17 Oct 2024 13:13:02 +0530 Subject: [PATCH 09/11] Removed Packagist testcase --- test/unit/lib/fetchTests.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 53019c63..98dfbdd0 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -59,17 +59,6 @@ describe('CallFetch', () => { checkHeader(requests[0].headers) }) - it('checks if the default header user-agent is present in packagist components', async () => { - const path = '/api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001' - const endpointMock = await mockServer.forGet(path).thenReply(200, 'success') - - await callFetch({ - url: mockServer.urlFor(path) - }) - const requests = await endpointMock.getSeenRequests() - checkHeader(requests[0].headers) - }) - it('checks if the full response is fetched', async () => { const path = '/registry.npmjs.com/redis/0.1.0' const expected = fs.readFileSync('test/fixtures/fetch/redis-0.1.0.json') From b900709cdbae622833e8bfd63bef707a89b4f0b9 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Mon, 21 Oct 2024 11:23:51 +0530 Subject: [PATCH 10/11] Replace user-agent with defaultHeader --- test/unit/providers/fetch/mavenBasedFetchTests.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/unit/providers/fetch/mavenBasedFetchTests.js b/test/unit/providers/fetch/mavenBasedFetchTests.js index 26844903..be78a673 100644 --- a/test/unit/providers/fetch/mavenBasedFetchTests.js +++ b/test/unit/providers/fetch/mavenBasedFetchTests.js @@ -2,8 +2,15 @@ const { expect } = require('chai') const MavenBasedFetch = require('../../../../providers/fetch/mavenBasedFetch') const mockttp = require('mockttp') const sinon = require('sinon') +const { defaultHeaders } = require('../../../../lib/fetch') const Request = require('../../../../ghcrawler').request +function checkHeader(headers) { + for (const [key, value] of Object.entries(defaultHeaders)) { + expect(headers).to.have.property(key.toLowerCase()).that.equals(value) + } +} + describe('MavenBasedFetch', () => { describe('find contained file stat', () => { it('file contained in root dir', async () => { @@ -47,7 +54,7 @@ describe('MavenBasedFetch', () => { it('should check for default header in any request', async () => { await handler.handle(new Request('test', 'cd:/maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16')) const requests = await endpointMock.getSeenRequests() - expect(requests[0].headers).to.include({ 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' }) + checkHeader(requests[0].headers) }) }) }) From edc902d815853feb6978c4900e512e2e3ccf0ac6 Mon Sep 17 00:00:00 2001 From: yashkohli88 Date: Wed, 30 Oct 2024 19:03:27 +0530 Subject: [PATCH 11/11] Update function name for improved clarity and readability --- test/unit/lib/fetchTests.js | 12 ++++++------ test/unit/providers/fetch/mavenBasedFetchTests.js | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js index 98dfbdd0..114cc3b5 100644 --- a/test/unit/lib/fetchTests.js +++ b/test/unit/lib/fetchTests.js @@ -4,7 +4,7 @@ const { expect } = require('chai') const fs = require('fs') const mockttp = require('mockttp') -function checkHeader(headers) { +function checkDefaultHeaders(headers) { for (const [key, value] of Object.entries(defaultHeaders)) { expect(headers).to.have.property(key.toLowerCase()).that.equals(value) } @@ -42,7 +42,7 @@ describe('CallFetch', () => { } }) const requests = await endpointMock.getSeenRequests() - checkHeader(requests[0].headers) + checkDefaultHeaders(requests[0].headers) expect(requests[0].headers).to.include({ accept: 'text/html' }) }) @@ -56,7 +56,7 @@ describe('CallFetch', () => { json: true }) const requests = await endpointMock.getSeenRequests() - checkHeader(requests[0].headers) + checkDefaultHeaders(requests[0].headers) }) it('checks if the full response is fetched', async () => { @@ -131,9 +131,9 @@ describe('CallFetch', () => { const requests = await endpointMock.getSeenRequests() expect(requests.length).to.equal(2) expect(requests[0].url).to.equal(url) - checkHeader(requests[0].headers) + checkDefaultHeaders(requests[0].headers) expect(requests[1].url).to.equal(url) - checkHeader(requests[1].headers) + checkDefaultHeaders(requests[1].headers) }) it('checks if the response is text with uri option in GET request', async () => { @@ -166,7 +166,7 @@ describe('CallFetch', () => { expect(json).to.deep.equal({ test: 'test' }) expect(requests[0].headers).to.include({ 'x-crawler': 'secret' }) //Check for the default header value - checkHeader(requests[0].headers) + checkDefaultHeaders(requests[0].headers) }) describe('test simple', () => { diff --git a/test/unit/providers/fetch/mavenBasedFetchTests.js b/test/unit/providers/fetch/mavenBasedFetchTests.js index be78a673..c48dc7ef 100644 --- a/test/unit/providers/fetch/mavenBasedFetchTests.js +++ b/test/unit/providers/fetch/mavenBasedFetchTests.js @@ -5,7 +5,7 @@ const sinon = require('sinon') const { defaultHeaders } = require('../../../../lib/fetch') const Request = require('../../../../ghcrawler').request -function checkHeader(headers) { +function checkDefaultHeaders(headers) { for (const [key, value] of Object.entries(defaultHeaders)) { expect(headers).to.have.property(key.toLowerCase()).that.equals(value) } @@ -54,7 +54,7 @@ describe('MavenBasedFetch', () => { it('should check for default header in any request', async () => { await handler.handle(new Request('test', 'cd:/maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16')) const requests = await endpointMock.getSeenRequests() - checkHeader(requests[0].headers) + checkDefaultHeaders(requests[0].headers) }) }) })