fix(http): compute checksum after download (#1196)

containerbase · Jul 13, 2023 · a77fd6f · a77fd6f
1 parent 3b7b84a
commit a77fd6f
Show file tree

Hide file tree

Showing 7 changed files with 130 additions and 25 deletions.
diff --git a/package.json b/package.json
@@ -32,7 +32,6 @@
     "clipanion": "3.2.1",
     "execa": "7.1.1",
     "got": "13.0.0",
-    "hasha": "5.2.2",
     "inversify": "6.0.1",
     "pino": "8.14.1",
     "pino-pretty": "10.0.1",
@@ -78,6 +77,7 @@
     "semantic-release": "21.0.7",
     "shelljs": "0.8.5",
     "tsx": "3.12.7",
+    "type-fest": "3.13.0",
     "typescript": "5.1.6",
     "vite-tsconfig-paths": "4.2.0",
     "vitest": "0.33.0"

diff --git a/src/cli/services/http.service.spec.ts b/src/cli/services/http.service.spec.ts
@@ -3,6 +3,7 @@ import type { Container } from 'inversify';
 import { beforeEach, describe, expect, test } from 'vitest';
 import { HttpService, rootContainer } from '.';
 import { scope } from '~test/http-mock';
+import { cacheFile } from '~test/path';
 
 const baseUrl = 'https://example.com';
 describe('http.service', () => {
@@ -20,6 +21,7 @@ describe('http.service', () => {
 
   test('throws', async () => {
     scope(baseUrl).get('/fail.txt').times(6).reply(404);
+
     const http = child.get(HttpService);
 
     await expect(
@@ -30,18 +32,60 @@ describe('http.service', () => {
     ).rejects.toThrow();
   });
 
+  test('throws with checksum', async () => {
+    scope(baseUrl).get('/checksum.txt').thrice().reply(200, 'ok');
+
+    const http = child.get(HttpService);
+    const expectedChecksum = 'invalid';
+    const checksumType = 'sha256';
+
+    await expect(
+      http.download({
+        url: `${baseUrl}/checksum.txt`,
+        expectedChecksum,
+        checksumType,
+      })
+    ).rejects.toThrow();
+  });
+
   test('download', async () => {
     scope(baseUrl).get('/test.txt').reply(200, 'ok');
 
     const http = child.get(HttpService);
-
-    expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(
-      `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt`
+    const expected = cacheFile(
+      `d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt`
     );
+
+    expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(expected);
     // uses cache
-    expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(
-      `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt`
+    expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe(expected);
+  });
+
+  test('download with checksum', async () => {
+    scope(baseUrl).get('/test.txt').reply(200, 'https://example.com/test.txt');
+
+    const http = child.get(HttpService);
+    const expectedChecksum =
+      'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020';
+    const expected = cacheFile(
+      `d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt`
     );
+
+    expect(
+      await http.download({
+        url: `${baseUrl}/test.txt`,
+        expectedChecksum,
+        checksumType: 'sha256',
+      })
+    ).toBe(expected);
+    // uses cache
+    expect(
+      await http.download({
+        url: `${baseUrl}/test.txt`,
+        expectedChecksum,
+        checksumType: 'sha256',
+      })
+    ).toBe(expected);
   });
 
   test('replaces url', async () => {
@@ -51,13 +95,16 @@ describe('http.service', () => {
     env.URL_REPLACE_0_TO = 'https://example.org';
 
     const http = child.get(HttpService);
+    const expected = cacheFile(
+      `f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt`
+    );
 
     expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe(
-      `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt`
+      expected
     );
     // uses cache
     expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe(
-      `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt`
+      expected
     );
   });
 });
diff --git a/src/cli/services/http.service.ts b/src/cli/services/http.service.ts
@@ -3,9 +3,9 @@ import { mkdir, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { got } from 'got';
-import hasha from 'hasha';
 import { inject, injectable } from 'inversify';
 import { logger } from '../utils';
+import { hash, hashFile } from '../utils/hash';
 import { EnvService } from './env.service';
 import { PathService } from './path.service';
 
@@ -36,7 +36,7 @@ export class HttpService {
     checksumType,
     fileName,
   }: HttpDownloadConfig): Promise<string> {
-    const urlChecksum = hasha(url, { algorithm: 'sha256' });
+    const urlChecksum = hash(url, 'sha256');
 
     const cacheDir = this.envSvc.cacheDir ?? this.pathSvc.tmpDir;
     const cachePath = join(cacheDir, urlChecksum);
@@ -46,9 +46,7 @@ export class HttpService {
 
     if (await this.pathSvc.fileExists(filePath)) {
       if (expectedChecksum && checksumType) {
-        const actualChecksum = await hasha.fromFile(filePath, {
-          algorithm: checksumType,
-        });
+        const actualChecksum = await hashFile(filePath, checksumType);
 
         if (actualChecksum === expectedChecksum) {
           return filePath;
@@ -70,6 +68,19 @@ export class HttpService {
     for (const run of [1, 2, 3]) {
       try {
         await pipeline(got.stream(nUrl), createWriteStream(filePath));
+        if (expectedChecksum && checksumType) {
+          const actualChecksum = await hashFile(filePath, checksumType);
+
+          if (actualChecksum === expectedChecksum) {
+            return filePath;
+          } else {
+            logger.debug(
+              { url, expectedChecksum, actualChecksum, checksumType },
+              'checksum mismatch'
+            );
+            throw new Error('checksum mismatch');
+          }
+        }
         return filePath;
       } catch (err) {
         if (run === 3) {

diff --git a/src/cli/utils/hash.spec.ts b/src/cli/utils/hash.spec.ts
@@ -0,0 +1,20 @@
+import fs from 'node:fs/promises';
+import { env } from 'node:process';
+import { describe, expect, test } from 'vitest';
+import { hash, hashFile } from './hash';
+
+describe('hash', () => {
+  test('should hash data with sha256', () => {
+    expect(hash('https://example.com/test.txt', 'sha256')).toBe(
+      'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020'
+    );
+  });
+
+  test('should hash file with sha256', async () => {
+    const file = `${env.CONTAINERBASE_CACHE_DIR}/test.txt`;
+    await fs.writeFile(file, 'https://example.com/test.txt');
+    expect(await hashFile(file, 'sha256')).toBe(
+      'd1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020'
+    );
+  });
+});
diff --git a/src/cli/utils/hash.ts b/src/cli/utils/hash.ts
@@ -0,0 +1,24 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import type { LiteralUnion } from 'type-fest';
+
+export type AlgorithmName = LiteralUnion<
+  'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512',
+  string
+>;
+
+export function hash(data: string | Buffer, algorithm: AlgorithmName): string {
+  const hash = crypto.createHash(algorithm);
+  hash.update(data);
+  return hash.digest('hex');
+}
+
+export async function hashFile(
+  file: string,
+  algorithm: AlgorithmName
+): Promise<string> {
+  const data = await fs.readFile(file);
+  const hash = crypto.createHash(algorithm);
+  hash.update(data);
+  return hash.digest('hex');
+}
diff --git a/test/path.ts b/test/path.ts
@@ -0,0 +1,6 @@
+import { sep } from 'node:path';
+import { env } from 'node:process';
+
+export function cacheFile(path: string): string {
+  return `${env.CONTAINERBASE_CACHE_DIR}/${path}`.replace(/\/+/g, sep);
+}
diff --git a/yarn.lock b/yarn.lock
@@ -2706,7 +2706,6 @@ __metadata:
     eslint-plugin-typescript-enum: 2.1.0
     execa: 7.1.1
     got: 13.0.0
-    hasha: 5.2.2
     husky: 8.0.3
     inversify: 6.0.1
     lint-staged: 13.2.3
@@ -2725,6 +2724,7 @@ __metadata:
     tar: 6.1.15
     tsx: 3.12.7
     typanion: 3.13.0
+    type-fest: 3.13.0
     typescript: 5.1.6
     vite-tsconfig-paths: 4.2.0
     vitest: 0.33.0
@@ -4533,16 +4533,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"hasha@npm:5.2.2":
-  version: 5.2.2
-  resolution: "hasha@npm:5.2.2"
-  dependencies:
-    is-stream: ^2.0.0
-    type-fest: ^0.8.0
-  checksum: 06cc474bed246761ff61c19d629977eb5f53fa817be4313a255a64ae0f433e831a29e83acb6555e3f4592b348497596f1d1653751008dda4f21c9c21ca60ac5a
-  languageName: node
-  linkType: hard
-
 "help-me@npm:^4.0.1":
   version: 4.2.0
   resolution: "help-me@npm:4.2.0"
@@ -8835,6 +8825,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"type-fest@npm:3.13.0":
+  version: 3.13.0
+  resolution: "type-fest@npm:3.13.0"
+  checksum: f7be142ae1ad0582eafd52d085350799c8cd918c15455896a06c82c147b61f8cea58892bedf1348943478e37740562219375b3b59fd855db99cd2b2766510f98
+  languageName: node
+  linkType: hard
+
 "type-fest@npm:^0.18.0":
   version: 0.18.1
   resolution: "type-fest@npm:0.18.1"
@@ -8863,7 +8860,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"type-fest@npm:^0.8.0, type-fest@npm:^0.8.1":
+"type-fest@npm:^0.8.1":
   version: 0.8.1
   resolution: "type-fest@npm:0.8.1"
   checksum: d61c4b2eba24009033ae4500d7d818a94fd6d1b481a8111612ee141400d5f1db46f199c014766b9fa9b31a6a7374d96fc748c6d688a78a3ce5a33123839becb7