From a77fd6ffe0487159fe4650dfcc02ef9ebe33d404 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 13 Jul 2023 22:28:52 +0200 Subject: [PATCH] fix(http): compute checksum after download (#1196) --- package.json | 2 +- src/cli/services/http.service.spec.ts | 61 ++++++++++++++++++++++++--- src/cli/services/http.service.ts | 21 ++++++--- src/cli/utils/hash.spec.ts | 20 +++++++++ src/cli/utils/hash.ts | 24 +++++++++++ test/path.ts | 6 +++ yarn.lock | 21 ++++----- 7 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 src/cli/utils/hash.spec.ts create mode 100644 src/cli/utils/hash.ts create mode 100644 test/path.ts diff --git a/package.json b/package.json index b0fe30d81..2b539d2e0 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "clipanion": "3.2.1", "execa": "7.1.1", "got": "13.0.0", - "hasha": "5.2.2", "inversify": "6.0.1", "pino": "8.14.1", "pino-pretty": "10.0.1", @@ -78,6 +77,7 @@ "semantic-release": "21.0.7", "shelljs": "0.8.5", "tsx": "3.12.7", + "type-fest": "3.13.0", "typescript": "5.1.6", "vite-tsconfig-paths": "4.2.0", "vitest": "0.33.0" diff --git a/src/cli/services/http.service.spec.ts b/src/cli/services/http.service.spec.ts index 83cbea8cf..e3be08dfc 100644 --- a/src/cli/services/http.service.spec.ts +++ b/src/cli/services/http.service.spec.ts @@ -3,6 +3,7 @@ import type { Container } from 'inversify'; import { beforeEach, describe, expect, test } from 'vitest'; import { HttpService, rootContainer } from '.'; import { scope } from '~test/http-mock'; +import { cacheFile } from '~test/path'; const baseUrl = 'https://example.com'; describe('http.service', () => { @@ -20,6 +21,7 @@ describe('http.service', () => { test('throws', async () => { scope(baseUrl).get('/fail.txt').times(6).reply(404); + const http = child.get(HttpService); await expect( @@ -30,18 +32,60 @@ describe('http.service', () => { ).rejects.toThrow(); }); + test('throws with checksum', async () => { + scope(baseUrl).get('/checksum.txt').thrice().reply(200, 'ok'); + + const http = child.get(HttpService); + const expectedChecksum = 'invalid'; + const checksumType = 'sha256'; + + await expect( + http.download({ + url: `${baseUrl}/checksum.txt`, + expectedChecksum, + checksumType, + }) + ).rejects.toThrow(); + }); + test('download', async () => { scope(baseUrl).get('/test.txt').reply(200, 'ok'); const http = child.get(HttpService); - - expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + const expected = cacheFile( + `d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` ); + + expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(expected); // uses cache - expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(expected); + }); + + test('download with checksum', async () => { + scope(baseUrl).get('/test.txt').reply(200, 'https://example.com/test.txt'); + + const http = child.get(HttpService); + const expectedChecksum = + 'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020'; + const expected = cacheFile( + `d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` ); + + expect( + await http.download({ + url: `${baseUrl}/test.txt`, + expectedChecksum, + checksumType: 'sha256', + }) + ).toBe(expected); + // uses cache + expect( + await http.download({ + url: `${baseUrl}/test.txt`, + expectedChecksum, + checksumType: 'sha256', + }) + ).toBe(expected); }); test('replaces url', async () => { @@ -51,13 +95,16 @@ describe('http.service', () => { env.URL_REPLACE_0_TO = 'https://example.org'; const http = child.get(HttpService); + const expected = cacheFile( + `f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt` + ); expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt` + expected ); // uses cache expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt` + expected ); }); }); diff --git a/src/cli/services/http.service.ts b/src/cli/services/http.service.ts index dbe2908ad..85c485007 100644 --- a/src/cli/services/http.service.ts +++ b/src/cli/services/http.service.ts @@ -3,9 +3,9 @@ import { mkdir, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { got } from 'got'; -import hasha from 'hasha'; import { inject, injectable } from 'inversify'; import { logger } from '../utils'; +import { hash, hashFile } from '../utils/hash'; import { EnvService } from './env.service'; import { PathService } from './path.service'; @@ -36,7 +36,7 @@ export class HttpService { checksumType, fileName, }: HttpDownloadConfig): Promise { - const urlChecksum = hasha(url, { algorithm: 'sha256' }); + const urlChecksum = hash(url, 'sha256'); const cacheDir = this.envSvc.cacheDir ?? this.pathSvc.tmpDir; const cachePath = join(cacheDir, urlChecksum); @@ -46,9 +46,7 @@ export class HttpService { if (await this.pathSvc.fileExists(filePath)) { if (expectedChecksum && checksumType) { - const actualChecksum = await hasha.fromFile(filePath, { - algorithm: checksumType, - }); + const actualChecksum = await hashFile(filePath, checksumType); if (actualChecksum === expectedChecksum) { return filePath; @@ -70,6 +68,19 @@ export class HttpService { for (const run of [1, 2, 3]) { try { await pipeline(got.stream(nUrl), createWriteStream(filePath)); + if (expectedChecksum && checksumType) { + const actualChecksum = await hashFile(filePath, checksumType); + + if (actualChecksum === expectedChecksum) { + return filePath; + } else { + logger.debug( + { url, expectedChecksum, actualChecksum, checksumType }, + 'checksum mismatch' + ); + throw new Error('checksum mismatch'); + } + } return filePath; } catch (err) { if (run === 3) { diff --git a/src/cli/utils/hash.spec.ts b/src/cli/utils/hash.spec.ts new file mode 100644 index 000000000..c2acbdb03 --- /dev/null +++ b/src/cli/utils/hash.spec.ts @@ -0,0 +1,20 @@ +import fs from 'node:fs/promises'; +import { env } from 'node:process'; +import { describe, expect, test } from 'vitest'; +import { hash, hashFile } from './hash'; + +describe('hash', () => { + test('should hash data with sha256', () => { + expect(hash('https://example.com/test.txt', 'sha256')).toBe( + 'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020' + ); + }); + + test('should hash file with sha256', async () => { + const file = `${env.CONTAINERBASE_CACHE_DIR}/test.txt`; + await fs.writeFile(file, 'https://example.com/test.txt'); + expect(await hashFile(file, 'sha256')).toBe( + 'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020' + ); + }); +}); diff --git a/src/cli/utils/hash.ts b/src/cli/utils/hash.ts new file mode 100644 index 000000000..3c568a23a --- /dev/null +++ b/src/cli/utils/hash.ts @@ -0,0 +1,24 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import type { LiteralUnion } from 'type-fest'; + +export type AlgorithmName = LiteralUnion< + 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512', + string +>; + +export function hash(data: string | Buffer, algorithm: AlgorithmName): string { + const hash = crypto.createHash(algorithm); + hash.update(data); + return hash.digest('hex'); +} + +export async function hashFile( + file: string, + algorithm: AlgorithmName +): Promise { + const data = await fs.readFile(file); + const hash = crypto.createHash(algorithm); + hash.update(data); + return hash.digest('hex'); +} diff --git a/test/path.ts b/test/path.ts new file mode 100644 index 000000000..1efaaf9ee --- /dev/null +++ b/test/path.ts @@ -0,0 +1,6 @@ +import { sep } from 'node:path'; +import { env } from 'node:process'; + +export function cacheFile(path: string): string { + return `${env.CONTAINERBASE_CACHE_DIR}/${path}`.replace(/\/+/g, sep); +} diff --git a/yarn.lock b/yarn.lock index 3fe21d37d..cb44c05cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2706,7 +2706,6 @@ __metadata: eslint-plugin-typescript-enum: 2.1.0 execa: 7.1.1 got: 13.0.0 - hasha: 5.2.2 husky: 8.0.3 inversify: 6.0.1 lint-staged: 13.2.3 @@ -2725,6 +2724,7 @@ __metadata: tar: 6.1.15 tsx: 3.12.7 typanion: 3.13.0 + type-fest: 3.13.0 typescript: 5.1.6 vite-tsconfig-paths: 4.2.0 vitest: 0.33.0 @@ -4533,16 +4533,6 @@ __metadata: languageName: node linkType: hard -"hasha@npm:5.2.2": - version: 5.2.2 - resolution: "hasha@npm:5.2.2" - dependencies: - is-stream: ^2.0.0 - type-fest: ^0.8.0 - checksum: 06cc474bed246761ff61c19d629977eb5f53fa817be4313a255a64ae0f433e831a29e83acb6555e3f4592b348497596f1d1653751008dda4f21c9c21ca60ac5a - languageName: node - linkType: hard - "help-me@npm:^4.0.1": version: 4.2.0 resolution: "help-me@npm:4.2.0" @@ -8835,6 +8825,13 @@ __metadata: languageName: node linkType: hard +"type-fest@npm:3.13.0": + version: 3.13.0 + resolution: "type-fest@npm:3.13.0" + checksum: f7be142ae1ad0582eafd52d085350799c8cd918c15455896a06c82c147b61f8cea58892bedf1348943478e37740562219375b3b59fd855db99cd2b2766510f98 + languageName: node + linkType: hard + "type-fest@npm:^0.18.0": version: 0.18.1 resolution: "type-fest@npm:0.18.1" @@ -8863,7 +8860,7 @@ __metadata: languageName: node linkType: hard -"type-fest@npm:^0.8.0, type-fest@npm:^0.8.1": +"type-fest@npm:^0.8.1": version: 0.8.1 resolution: "type-fest@npm:0.8.1" checksum: d61c4b2eba24009033ae4500d7d818a94fd6d1b481a8111612ee141400d5f1db46f199c014766b9fa9b31a6a7374d96fc748c6d688a78a3ce5a33123839becb7