Skip to content

Commit 373d341

Browse files
bodinsamuelbodinsamuel
authored
feat(jsDelivr): move code, add tests, preload data correctly (#384)
1 parent 7cd0765 commit 373d341

File tree

8 files changed

+370
-53
lines changed

8 files changed

+370
-53
lines changed

‎src/__tests__/__snapshots__/config.test.js.snap‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ Object {
178178
},
179179
],
180180
"jsDelivrHitsEndpoint": "https://data.jsdelivr.com/v1/stats/packages/npm/month/all",
181+
"jsDelivrPackageEndpoint": "https://data.jsdelivr.com/v1/package/npm",
181182
"maxObjSize": 450000,
182183
"npmDownloadsEndpoint": "https://api.npmjs.org/downloads",
183184
"npmRegistryEndpoint": "https://replicate.npmjs.com/registry",

‎src/config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const defaultConfig = {
88
npmRootEndpoint: 'https://registry.npmjs.org',
99
jsDelivrHitsEndpoint:
1010
'https://data.jsdelivr.com/v1/stats/packages/npm/month/all',
11+
jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm',
1112
unpkgRoot: 'https://unpkg.com',
1213
maxObjSize: 450000,
1314
popularDownloadsRatio: 0.005,

‎src/index.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import config from './config.js';
88
import * as npm from './npm/index.js';
99
import log from './log.js';
1010
import datadog from './datadog.js';
11-
import { loadHits } from './jsDelivr.js';
11+
import * as jsDelivr from './jsDelivr/index.js';
1212

1313
log.info('🗿 npm ↔️ Algolia replication starts ⛷ 🐌 🛰');
1414

@@ -31,6 +31,9 @@ async function main() {
3131
await setSettings(bootstrapIndex);
3232
datadog.timing('main.init_algolia', Date.now() - start);
3333

34+
// Preload some useful data
35+
await jsDelivr.loadHits();
36+
3437
// then we run the bootstrap
3538
// after a bootstrap is done, it's moved to main (with settings)
3639
// if it was already finished, we will set the settings on the main index
@@ -106,8 +109,6 @@ async function bootstrap(state) {
106109
return state;
107110
}
108111

109-
await loadHits();
110-
111112
const { seq, nbDocs: totalDocs } = await npm.getInfo();
112113
if (!state.bootstrapLastId) {
113114
// Start from 0

‎src/jsDelivr.js‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎src/jsDelivr/__test__/__snapshots__/index.test.js.snap‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`files getAllFilesList() should get a flat list of files 1`] = `
4+
Array [
5+
Array [
6+
Object {
7+
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
8+
"name": "/bin/jest.js",
9+
"size": 343,
10+
"time": "1985-10-26T08:15:00.000Z",
11+
},
12+
Object {
13+
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
14+
"name": "/build/jest.d.ts",
15+
"size": 291,
16+
"time": "1985-10-26T08:15:00.000Z",
17+
},
18+
Object {
19+
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
20+
"name": "/build/jest.d.ts.map",
21+
"size": 171,
22+
"time": "1985-10-26T08:15:00.000Z",
23+
},
24+
Object {
25+
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
26+
"name": "/build/jest.js",
27+
"size": 1030,
28+
"time": "1985-10-26T08:15:00.000Z",
29+
},
30+
Object {
31+
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
32+
"name": "/LICENSE",
33+
"size": 1099,
34+
"time": "1985-10-26T08:15:00.000Z",
35+
},
36+
Object {
37+
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
38+
"name": "/package.json",
39+
"size": 925,
40+
"time": "1985-10-26T08:15:00.000Z",
41+
},
42+
Object {
43+
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
44+
"name": "/README.md",
45+
"size": 551,
46+
"time": "1985-10-26T08:15:00.000Z",
47+
},
48+
Object {
49+
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
50+
"name": "/tsconfig.json",
51+
"size": 162,
52+
"time": "1985-10-26T08:15:00.000Z",
53+
},
54+
Object {
55+
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
56+
"name": "/tsconfig.tsbuildinfo",
57+
"size": 220798,
58+
"time": "1985-10-26T08:15:00.000Z",
59+
},
60+
],
61+
]
62+
`;
63+
64+
exports[`files getAllFilesList() should get multiple flat list of files 1`] = `
65+
Array [
66+
Array [
67+
Object {
68+
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
69+
"name": "/bin/jest.js",
70+
"size": 343,
71+
"time": "1985-10-26T08:15:00.000Z",
72+
},
73+
Object {
74+
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
75+
"name": "/build/jest.d.ts",
76+
"size": 291,
77+
"time": "1985-10-26T08:15:00.000Z",
78+
},
79+
Object {
80+
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
81+
"name": "/build/jest.d.ts.map",
82+
"size": 171,
83+
"time": "1985-10-26T08:15:00.000Z",
84+
},
85+
Object {
86+
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
87+
"name": "/build/jest.js",
88+
"size": 1030,
89+
"time": "1985-10-26T08:15:00.000Z",
90+
},
91+
Object {
92+
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
93+
"name": "/LICENSE",
94+
"size": 1099,
95+
"time": "1985-10-26T08:15:00.000Z",
96+
},
97+
Object {
98+
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
99+
"name": "/package.json",
100+
"size": 925,
101+
"time": "1985-10-26T08:15:00.000Z",
102+
},
103+
Object {
104+
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
105+
"name": "/README.md",
106+
"size": 551,
107+
"time": "1985-10-26T08:15:00.000Z",
108+
},
109+
Object {
110+
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
111+
"name": "/tsconfig.json",
112+
"size": 162,
113+
"time": "1985-10-26T08:15:00.000Z",
114+
},
115+
Object {
116+
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
117+
"name": "/tsconfig.tsbuildinfo",
118+
"size": 220798,
119+
"time": "1985-10-26T08:15:00.000Z",
120+
},
121+
],
122+
Array [],
123+
]
124+
`;
125+
126+
exports[`files getFilesList() should get a flat list of files 1`] = `
127+
Array [
128+
Object {
129+
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
130+
"name": "/bin/jest.js",
131+
"size": 343,
132+
"time": "1985-10-26T08:15:00.000Z",
133+
},
134+
Object {
135+
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
136+
"name": "/build/jest.d.ts",
137+
"size": 291,
138+
"time": "1985-10-26T08:15:00.000Z",
139+
},
140+
Object {
141+
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
142+
"name": "/build/jest.d.ts.map",
143+
"size": 171,
144+
"time": "1985-10-26T08:15:00.000Z",
145+
},
146+
Object {
147+
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
148+
"name": "/build/jest.js",
149+
"size": 1030,
150+
"time": "1985-10-26T08:15:00.000Z",
151+
},
152+
Object {
153+
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
154+
"name": "/LICENSE",
155+
"size": 1099,
156+
"time": "1985-10-26T08:15:00.000Z",
157+
},
158+
Object {
159+
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
160+
"name": "/package.json",
161+
"size": 925,
162+
"time": "1985-10-26T08:15:00.000Z",
163+
},
164+
Object {
165+
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
166+
"name": "/README.md",
167+
"size": 551,
168+
"time": "1985-10-26T08:15:00.000Z",
169+
},
170+
Object {
171+
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
172+
"name": "/tsconfig.json",
173+
"size": 162,
174+
"time": "1985-10-26T08:15:00.000Z",
175+
},
176+
Object {
177+
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
178+
"name": "/tsconfig.tsbuildinfo",
179+
"size": 220798,
180+
"time": "1985-10-26T08:15:00.000Z",
181+
},
182+
]
183+
`;

‎src/jsDelivr/__test__/index.test.js‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import * as api from '../index.js';
2+
3+
describe('hits', () => {
4+
beforeAll(async () => {
5+
await api.loadHits();
6+
});
7+
describe('loadHits()', () => {
8+
it('should download all packages hits', () => {
9+
expect(api.hits.size).toBeGreaterThan(60000); // 66790 (2019-08)
10+
});
11+
12+
it('should get one hit', () => {
13+
expect(api.hits.get('jquery')).toBeGreaterThan(1000000000); // 1065750968 (2019-08)
14+
});
15+
16+
it('should not get one hit', () => {
17+
expect(api.hits.get('thispackagedoesnotexist')).toBe(undefined);
18+
});
19+
});
20+
21+
describe('getHits()', () => {
22+
it('should get one formatted hit', () => {
23+
expect(api.getHits(['jquery'])).toEqual([
24+
{
25+
jsDelivrHits: expect.any(Number),
26+
_searchInternal: {
27+
jsDelivrPopularity: expect.any(Number),
28+
},
29+
},
30+
]);
31+
});
32+
it('should get multiple formatted hits', () => {
33+
expect(api.getHits(['jquery', 'thispackagedoesnotexist'])).toEqual([
34+
{
35+
jsDelivrHits: expect.any(Number),
36+
_searchInternal: {
37+
jsDelivrPopularity: expect.any(Number),
38+
},
39+
},
40+
{
41+
jsDelivrHits: 0,
42+
_searchInternal: {
43+
jsDelivrPopularity: 0,
44+
},
45+
},
46+
]);
47+
});
48+
});
49+
});
50+
51+
describe('files', () => {
52+
describe('getFilesList()', () => {
53+
it('should get a flat list of files', async () => {
54+
const files = await api.getFilesList({ name: 'jest@24.8.0' });
55+
expect(files).toMatchSnapshot();
56+
});
57+
58+
it('should not get a files list', async () => {
59+
const files = await api.getFilesList({
60+
name: 'thispackagedoesnotexist@3.33.0',
61+
});
62+
expect(files).toEqual([]);
63+
});
64+
});
65+
66+
describe('getAllFilesList()', () => {
67+
it('should get a flat list of files', async () => {
68+
const files = await api.getAllFilesList([{ name: 'jest@24.8.0' }]);
69+
expect(files).toMatchSnapshot();
70+
});
71+
72+
it('should get multiple flat list of files', async () => {
73+
const files = await api.getAllFilesList([
74+
{
75+
name: 'jest@24.8.0',
76+
},
77+
{
78+
name: 'thispackagedoesnotexist@3.33.0',
79+
},
80+
]);
81+
expect(files).toMatchSnapshot();
82+
});
83+
});
84+
});

0 commit comments

Comments
 (0)