-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
hash.ts
122 lines (103 loc) · 4.02 KB
/
hash.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import crypto from 'crypto';
import extract from 'extract-zip';
import pMap from 'p-map';
import upath from 'upath';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
import type { TerraformBuild } from '../../../datasource/terraform-provider/types';
import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
import * as fs from '../../../util/fs';
import { ensureCacheDir } from '../../../util/fs';
import { Http } from '../../../util/http';
import { regEx } from '../../../util/regex';
export class TerraformProviderHash {
static http = new Http(TerraformProviderDatasource.id);
static terraformDatasource = new TerraformProviderDatasource();
static hashCacheTTL = 10080; // in minutes == 1 week
private static async hashFiles(files: string[]): Promise<string> {
const rootHash = crypto.createHash('sha256');
for (const file of files) {
// build for every file a line looking like "aaaaaaaaaaaaaaa file.txt\n"
const hash = crypto.createHash('sha256');
// a sha256sum displayed as lowercase hex string to root hash
const fileBuffer = await fs.readFile(file);
hash.update(fileBuffer);
rootHash.update(hash.digest('hex'));
// add double space, the filename and a new line char
rootHash.update(' ');
const fileName = file.replace(regEx(/^.*[\\/]/), '');
rootHash.update(fileName);
rootHash.update('\n');
}
return rootHash.digest('base64');
}
static async hashOfZipContent(
zipFilePath: string,
extractPath: string
): Promise<string> {
await extract(zipFilePath, { dir: extractPath });
const files = await fs.readdir(extractPath);
// the h1 hashing algorithms requires that the files are sorted by filename
const sortedFiles = files.sort((a, b) => a.localeCompare(b));
const filesWithPath = sortedFiles.map((file) => `${extractPath}/${file}`);
const result = await TerraformProviderHash.hashFiles(filesWithPath);
// delete extracted files
await fs.rm(extractPath, { recursive: true });
return result;
}
@cache({
namespace: `datasource-${TerraformProviderDatasource.id}-build-hashes`,
key: (build: TerraformBuild) => build.url,
ttlMinutes: TerraformProviderHash.hashCacheTTL,
})
static async calculateSingleHash(
build: TerraformBuild,
cacheDir: string
): Promise<string> {
const downloadFileName = upath.join(cacheDir, build.filename);
const extractPath = upath.join(cacheDir, 'extract', build.filename);
logger.trace(
`Downloading archive and generating hash for ${build.name}-${build.version}...`
);
const readStream = TerraformProviderHash.http.stream(build.url);
const writeStream = fs.createWriteStream(downloadFileName);
try {
await fs.pipeline(readStream, writeStream);
const hash = await this.hashOfZipContent(downloadFileName, extractPath);
logger.trace(
{ hash },
`Generated hash for ${build.name}-${build.version}`
);
return hash;
} finally {
// delete zip file
await fs.unlink(downloadFileName);
}
}
static async calculateHashes(builds: TerraformBuild[]): Promise<string[]> {
const cacheDir = await ensureCacheDir('./others/terraform');
// for each build download ZIP, extract content and generate hash for all containing files
return pMap(
builds,
(build) => this.calculateSingleHash(build, cacheDir),
{ concurrency: 4 } // allow to look up 4 builds for this version in parallel
);
}
static async createHashes(
registryURL: string,
repository: string,
version: string
): Promise<string[]> {
const builds = await TerraformProviderHash.terraformDatasource.getBuilds(
registryURL,
repository,
version
);
if (!builds) {
return null;
}
const hashes = await TerraformProviderHash.calculateHashes(builds);
// sorting the hash alphabetically as terraform does this as well
return hashes.sort().map((hash) => `h1:${hash}`);
}
}