From 715970feed13a5a0a7503121aa14d1ba5fa45d14 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 18 Jun 2023 18:12:25 +1000 Subject: [PATCH] feat: Add hash of `.cargo/config.toml` to key (#149) Also: - Add and use `sort_and_uniq` to make sure `globFile` resulting and `keyFiles` does not contain duplicates. - Only returns regular file in function `globFile` Signed-off-by: Jiahao XU --- dist/restore/index.js | 39 ++++++++++++++++++++++++++------------- dist/save/index.js | 39 ++++++++++++++++++++++++++------------- src/config.ts | 43 ++++++++++++++++++++++++++++++------------- 3 files changed, 82 insertions(+), 39 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index 2eed732..7131601 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -64194,15 +64194,13 @@ class CacheConfig { workspaces.push(new Workspace(root, target)); } self.workspaces = workspaces; - let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml"); + let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml"); const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed hasher = external_crypto_default().createHash("sha1"); for (const workspace of workspaces) { const root = workspace.root; - keyFiles.push(...(await globFiles(`${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`))); - const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`)) - .filter(file => !external_fs_default().statSync(file).isDirectory()); - cargo_manifests.sort((a, b) => a.localeCompare(b)); + keyFiles.push(...(await globFiles(`${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`))); + const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`)); for (const cargo_manifest of cargo_manifests) { try { const content = await promises_default().readFile(cargo_manifest, { encoding: 'utf8' }); @@ -64233,9 +64231,7 @@ class CacheConfig { keyFiles.push(cargo_manifest); } } - const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`)) - .filter(file => !external_fs_default().statSync(file).isDirectory()); - cargo_locks.sort((a, b) => a.localeCompare(b)); + const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`)); for (const cargo_lock of cargo_locks) { try { const content = await promises_default().readFile(cargo_lock, { encoding: 'utf8' }); @@ -64259,8 +64255,7 @@ class CacheConfig { } } } - keyFiles = keyFiles.filter(file => !external_fs_default().statSync(file).isDirectory()); - keyFiles.sort((a, b) => a.localeCompare(b)); + keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { for await (const chunk of external_fs_default().createReadStream(file)) { hasher.update(chunk); @@ -64268,8 +64263,7 @@ class CacheConfig { } let lockHash = digest(hasher); keyFiles.push(...parsedKeyFiles); - keyFiles.sort((a, b) => a.localeCompare(b)); - self.keyFiles = keyFiles; + self.keyFiles = sort_and_uniq(keyFiles); key += `-${lockHash}`; self.cacheKey = key; self.cachePaths = [config_CARGO_HOME]; @@ -64372,7 +64366,26 @@ async function globFiles(pattern) { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - return await globber.glob(); + // fs.statSync resolve the symbolic link and returns stat for the + // file it pointed to, so isFile would make sure the resolved + // file is actually a regular file. + return (await globber.glob()).filter(file => external_fs_default().statSync(file).isFile()); +} +function sort_and_uniq(a) { + return a + .sort((a, b) => a.localeCompare(b)) + .reduce((accumulator, currentValue) => { + const len = accumulator.length; + // If accumulator is empty or its last element != currentValue + // Since array is already sorted, elements with the same value + // are grouped together to be continugous in space. + // + // If currentValue != last element, then it must be unique. + if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) { + accumulator.push(currentValue); + } + return accumulator; + }, []); } function sort_object(o) { if (Array.isArray(o)) { diff --git a/dist/save/index.js b/dist/save/index.js index 255ba83..c6879ff 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -64194,15 +64194,13 @@ class CacheConfig { workspaces.push(new Workspace(root, target)); } self.workspaces = workspaces; - let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml"); + let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml"); const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed hasher = external_crypto_default().createHash("sha1"); for (const workspace of workspaces) { const root = workspace.root; - keyFiles.push(...(await globFiles(`${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`))); - const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`)) - .filter(file => !external_fs_default().statSync(file).isDirectory()); - cargo_manifests.sort((a, b) => a.localeCompare(b)); + keyFiles.push(...(await globFiles(`${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`))); + const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`)); for (const cargo_manifest of cargo_manifests) { try { const content = await promises_default().readFile(cargo_manifest, { encoding: 'utf8' }); @@ -64233,9 +64231,7 @@ class CacheConfig { keyFiles.push(cargo_manifest); } } - const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`)) - .filter(file => !external_fs_default().statSync(file).isDirectory()); - cargo_locks.sort((a, b) => a.localeCompare(b)); + const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`)); for (const cargo_lock of cargo_locks) { try { const content = await promises_default().readFile(cargo_lock, { encoding: 'utf8' }); @@ -64259,8 +64255,7 @@ class CacheConfig { } } } - keyFiles = keyFiles.filter(file => !external_fs_default().statSync(file).isDirectory()); - keyFiles.sort((a, b) => a.localeCompare(b)); + keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { for await (const chunk of external_fs_default().createReadStream(file)) { hasher.update(chunk); @@ -64268,8 +64263,7 @@ class CacheConfig { } let lockHash = digest(hasher); keyFiles.push(...parsedKeyFiles); - keyFiles.sort((a, b) => a.localeCompare(b)); - self.keyFiles = keyFiles; + self.keyFiles = sort_and_uniq(keyFiles); key += `-${lockHash}`; self.cacheKey = key; self.cachePaths = [CARGO_HOME]; @@ -64372,7 +64366,26 @@ async function globFiles(pattern) { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - return await globber.glob(); + // fs.statSync resolve the symbolic link and returns stat for the + // file it pointed to, so isFile would make sure the resolved + // file is actually a regular file. + return (await globber.glob()).filter(file => external_fs_default().statSync(file).isFile()); +} +function sort_and_uniq(a) { + return a + .sort((a, b) => a.localeCompare(b)) + .reduce((accumulator, currentValue) => { + const len = accumulator.length; + // If accumulator is empty or its last element != currentValue + // Since array is already sorted, elements with the same value + // are grouped together to be continugous in space. + // + // If currentValue != last element, then it must be unique. + if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) { + accumulator.push(currentValue); + } + return accumulator; + }, []); } function sort_object(o) { if (Array.isArray(o)) { diff --git a/src/config.ts b/src/config.ts index 83a28a7..fc3d93e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -129,7 +129,7 @@ export class CacheConfig { } self.workspaces = workspaces; - let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml"); + let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml"); const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed hasher = crypto.createHash("sha1"); @@ -138,13 +138,11 @@ export class CacheConfig { const root = workspace.root; keyFiles.push( ...(await globFiles( - `${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`, + `${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`, )), ); - const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`)) - .filter(file => !fs.statSync(file).isDirectory()); - cargo_manifests.sort((a, b) => a.localeCompare(b)); + const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`)); for (const cargo_manifest of cargo_manifests) { try { @@ -182,9 +180,7 @@ export class CacheConfig { } } - const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`)) - .filter(file => !fs.statSync(file).isDirectory()); - cargo_locks.sort((a, b) => a.localeCompare(b)); + const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`)); for (const cargo_lock of cargo_locks) { try { @@ -212,8 +208,7 @@ export class CacheConfig { } } } - keyFiles = keyFiles.filter(file => !fs.statSync(file).isDirectory()); - keyFiles.sort((a, b) => a.localeCompare(b)); + keyFiles = sort_and_uniq(keyFiles); for (const file of keyFiles) { for await (const chunk of fs.createReadStream(file)) { @@ -224,8 +219,7 @@ export class CacheConfig { let lockHash = digest(hasher); keyFiles.push(...parsedKeyFiles); - keyFiles.sort((a, b) => a.localeCompare(b)); - self.keyFiles = keyFiles; + self.keyFiles = sort_and_uniq(keyFiles); key += `-${lockHash}`; self.cacheKey = key; @@ -348,7 +342,30 @@ async function globFiles(pattern: string): Promise { const globber = await glob.create(pattern, { followSymbolicLinks: false, }); - return await globber.glob(); + // fs.statSync resolve the symbolic link and returns stat for the + // file it pointed to, so isFile would make sure the resolved + // file is actually a regular file. + return (await globber.glob()).filter(file => fs.statSync(file).isFile()); +} + +function sort_and_uniq(a: string[]) { + return a + .sort((a, b) => a.localeCompare(b)) + .reduce( + (accumulator: string[], currentValue: string) => { + const len = accumulator.length; + // If accumulator is empty or its last element != currentValue + // Since array is already sorted, elements with the same value + // are grouped together to be continugous in space. + // + // If currentValue != last element, then it must be unique. + if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) { + accumulator.push(currentValue); + } + return accumulator; + }, + [] + ); } function sort_object(o: any): any {