Skip to content

Commit

Permalink
cache: when fetching from cache, add missing symlink for every checks…
Browse files Browse the repository at this point in the history
…um found in opam file
  • Loading branch information
kit-ty-kate authored and rjbou committed Jul 15, 2024
1 parent ea76734 commit d152299
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 97 deletions.
3 changes: 3 additions & 0 deletions master_changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ users)

## Install
* Fix package name display for no agreement conflicts [#6055 @rjbou - fix #6030]
* Make fetching an archive from cache add missing symlinks [#6068 @kit-ty-kate - fix #6064]

## Remove

Expand Down Expand Up @@ -93,6 +94,7 @@ users)
* Add opam 2.2.0 to the install scripts [#6062 @kit-ty-kate]

## Admin
* Make `opam admin cache` add missing symlinks [#6068 @kit-ty-kate - fix #6064]

## Opam installer

Expand Down Expand Up @@ -145,6 +147,7 @@ users)
## opam-client

## opam-repository
* `OpamRepository.fetch_from_cache`: when an archive is found, add a symlink (or copy) for the ones found in opam file but not in cache [#6068 @kit-ty-kate]

## opam-state
* `OpamStateConfig.opamroot_with_provenance`: restore previous behaviour to `OpamStateConfig.opamroot` for compatibility with third party code [#6047 @dra27]
Expand Down
11 changes: 6 additions & 5 deletions src/client/opamAdminCommand.ml
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,13 @@ let package_files_to_cache repo_root cache_dir cache_urls
OpamConsole.warning "[%s] no checksum, not caching"
(OpamConsole.colorise `green label);
Done errors
| best_chks :: _ ->
let cache_file =
OpamRepository.cache_file cache_dir best_chks
| _::_ ->
let cache_files =
List.map (OpamRepository.cache_file cache_dir) checksums
in
let error_opt =
if not recheck && OpamFilename.exists cache_file then Done None
if not recheck && List.for_all OpamFilename.exists cache_files then
Done None
else
OpamRepository.pull_file_to_cache label
~cache_urls ~cache_dir
Expand All @@ -190,7 +191,7 @@ let package_files_to_cache repo_root cache_dir cache_urls
let link =
OpamFilename.Op.(link_dir / OpamPackage.to_string nv // name)
in
OpamFilename.link ~relative:true ~target:cache_file ~link)
OpamFilename.link ~relative:true ~target:(List.hd cache_files) ~link)
link;
errors
in
Expand Down
47 changes: 27 additions & 20 deletions src/repository/opamRepository.ml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ let cache_file cache_dir checksum =
in
aux cache_dir (OpamHash.to_path checksum)

let link_files ~target f l =
List.iter (fun x ->
OpamFilename.link ~relative:true ~target ~link:(f x))
l

let fetch_from_cache =
let currently_downloading = ref [] in
let rec no_concurrent_dls key f x =
Expand Down Expand Up @@ -93,21 +98,27 @@ let fetch_from_cache =
failwith "Version control not allowed as cache URL"
in
try
let hit_file =
OpamStd.List.find_map (fun ck ->
match
List.fold_left (fun (hit, misses) ck ->
let f = cache_file cache_dir ck in
if OpamFilename.exists f then Some f else None)
checksums
in
if List.for_all
(fun ck -> OpamHash.check_file (OpamFilename.to_string hit_file) ck)
checksums
then Done (Up_to_date (hit_file, OpamUrl.empty))
else mismatch hit_file
if OpamFilename.exists f
then (Some f, misses)
else (hit, f :: misses))
(None, []) checksums
with
| None, _ -> raise Not_found
| Some hit_file, miss_files ->
if List.for_all
(fun ck -> OpamHash.check_file (OpamFilename.to_string hit_file) ck)
checksums
then begin
link_files ~target:hit_file Fun.id miss_files;
Done (Up_to_date (hit_file, OpamUrl.empty))
end else
mismatch hit_file
with Not_found -> match checksums with
| [] -> let m = "cache miss" in Done (Not_available (Some m, m))
| checksum::_ ->
(* Try all cache urls in order, but only the first checksum *)
| checksum::other_checksums ->
let local_file = cache_file cache_dir checksum in
let tmpfile = OpamFilename.add_extension local_file "tmp" in
let rec try_cache_dl = function
Expand All @@ -125,6 +136,7 @@ let fetch_from_cache =
checksums
then
(OpamFilename.move ~src:tmpfile ~dst:local_file;
link_files ~target:local_file (cache_file cache_dir) other_checksums;
Done (Result (local_file, root_cache_url)))
else mismatch tmpfile
in
Expand All @@ -151,14 +163,9 @@ let validate_and_add_to_cache label url cache_dir file checksums =
(let checksums = OpamHash.sort checksums in
match cache_dir, checksums with
| Some dir, best_chks :: others_chks ->
OpamFilename.copy ~src:file ~dst:(cache_file dir best_chks);
List.iter (fun checksum ->
let target = cache_file dir best_chks in
let link = cache_file dir checksum in
try
OpamFilename.link ~relative:true ~target ~link
with Sys_error _ -> ())
others_chks;
let target = cache_file dir best_chks in
OpamFilename.copy ~src:file ~dst:target;
link_files ~target (cache_file dir) others_chks;
| _ -> ());
true

Expand Down
67 changes: 31 additions & 36 deletions tests/reftests/admin-cache.unix.test
Original file line number Diff line number Diff line change
Expand Up @@ -147,40 +147,44 @@ amet : sha512 : exists
### # in cache, the archiv is the strongest one
### rm -rf cache/md5 cache/sha256
### opam admin cache | unordered
[dolor.1] synchronised (cached)
[ipsum.1] synchronised (cached)
[sit.1] synchronised (cached)
[amet.1] synchronised (cached)
Done.
### sh cache-analysi.sh
ipsum : md5 : not found
ipsum : sha256 : not found
ipsum : md5 : exists, link to sha512
ipsum : sha256 : exists, link to sha512
ipsum : sha512 : exists
--
dolor : md5 : not found
dolor : sha256 : not found
dolor : md5 : exists, link to sha512
dolor : sha256 : exists, link to sha512
dolor : sha512 : exists
--
sit : md5 : not found
sit : sha256 : not found
sit : md5 : exists, link to sha512
sit : sha256 : exists, link to sha512
sit : sha512 : exists
--
amet : md5 : not found
amet : sha256 : not found
amet : md5 : exists, link to sha512
amet : sha256 : exists, link to sha512
amet : sha512 : exists
### opam admin cache
Done.
### sh cache-analysi.sh
ipsum : md5 : not found
ipsum : sha256 : not found
ipsum : md5 : exists, link to sha512
ipsum : sha256 : exists, link to sha512
ipsum : sha512 : exists
--
dolor : md5 : not found
dolor : sha256 : not found
dolor : md5 : exists, link to sha512
dolor : sha256 : exists, link to sha512
dolor : sha512 : exists
--
sit : md5 : not found
sit : sha256 : not found
sit : md5 : exists, link to sha512
sit : sha256 : exists, link to sha512
sit : sha512 : exists
--
amet : md5 : not found
amet : sha256 : not found
amet : md5 : exists, link to sha512
amet : sha256 : exists, link to sha512
amet : sha512 : exists
### :::::::::::::::::::::::::::::::::::::::::::
### :III: remove strongest hash and recompute :
Expand Down Expand Up @@ -258,13 +262,13 @@ amet : sha512 : exists
### sed -i '/"sha512=.*/d' packages/dolor/dolor.1/opam
### sed -i '/"md5=.*/d' packages/sit/sit.1/opam
### sed -i '/"sha512=.*/d' packages/sit/sit.1/opam
### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 | "${IP_SHA512}" -> IP_SHA512
### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5
url.src: file://${BASEDIR}/arch-ipsum.tgz
url.checksum: md5=IP_MD5
### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256 | "${DL_SHA512}" -> DL_SHA512
### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256
url.src: file://${BASEDIR}/arch-dolor.tgz
url.checksum: md5=DL_MD5, sha256=DL_SHA256
### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_MD5}" -> ST_MD5 | "${ST_SHA256}" -> ST_SHA256 | "${ST_SHA512}" -> ST_SHA512
### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_SHA256}" -> ST_SHA256
url.src: file://${BASEDIR}/arch-sit.tgz
url.checksum: sha256=ST_SHA256
### rm -rf cache
Expand Down Expand Up @@ -294,7 +298,7 @@ amet : sha512 : exists
### opam admin add-hashes sha512 --package ipsum
### opam admin add-hashes sha512 --package dolor
### opam admin add-hashes md5 --package sit
[file://${BASEDIR}/arch-sit.tgz] downloaded from file://${BASEDIR}/cache
[file://${BASEDIR}/arch-sit.tgz] found in cache
### opam admin add-hashes sha512 --package sit
### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 | "${IP_SHA512}" -> IP_SHA512
url.src: file://${BASEDIR}/arch-ipsum.tgz
Expand Down Expand Up @@ -322,23 +326,23 @@ amet : md5 : exists, link to sha512
amet : sha256 : exists, link to sha512
amet : sha512 : exists
### : cache
### opam admin cache
### opam admin cache | unordered
[dolor.1] synchronised (cached)
[ipsum.1] synchronised (cached)
[sit.1] synchronised (cached)
Done.
### sh cache-analysi.sh
ipsum : md5 : exists
ipsum : sha256 : not found
ipsum : sha512 : not found
ipsum : sha512 : exists, link to md5
--
dolor : md5 : exists, link to sha256
dolor : sha256 : exists
dolor : sha512 : not found
dolor : sha512 : exists, link to md5
--
sit : md5 : not found
sit : md5 : exists, link to sha256
sit : sha256 : exists
sit : sha512 : not found
sit : sha512 : exists, link to sha256
--
amet : md5 : exists, link to sha512
amet : sha256 : exists, link to sha512
Expand All @@ -364,9 +368,6 @@ opam-version: "2.0"
MD5(arch-evil.tgz)= LR_MD5
### sh add-urls.sh lorem
### opam admin cache
[dolor.1] synchronised (cached)
[ipsum.1] synchronised (cached)
[sit.1] synchronised (cached)
[lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz)
Done.
### opam admin add-hashes sha256 --package lorem
Expand All @@ -380,29 +381,23 @@ lorem : sha256 : not found
lorem : sha512 : not found
### opam admin cache
[lorem.1] synchronised (cached)
[dolor.1] synchronised (cached)
[ipsum.1] synchronised (cached)
[sit.1] synchronised (cached)
Done.
### sh cache-analysi.sh
lorem : md5 : exists
lorem : sha256 : not found
lorem : sha256 : exists, link to md5
lorem : sha512 : not found
### mv arch-evil.tgz cache/md5/$PRE_LR_MD5/$LR_MD5
### opam admin add-hashes sha512 --package lorem
### sh cache-analysi.sh
lorem : md5 : exists
lorem : sha256 : not found
lorem : sha256 : exists, link to md5
lorem : sha512 : not found
### opam admin cache | "${LR_MD5}" -> LR_MD5 | "${LR_SHA256}" -> LR_SHA256 | "${LR_SHA512}" -> LR_SHA512
[ERROR] Conflicting file hashes, or broken or compromised cache!
- sha512=LR_SHA512 (MISMATCH)
- sha256=LR_SHA256 (MISMATCH)
- md5=LR_MD5 (match)

[dolor.1] synchronised (cached)
[ipsum.1] synchronised (cached)
[sit.1] synchronised (cached)
[lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz)
Done.
### sh cache-analysi.sh
Expand Down
Loading

0 comments on commit d152299

Please sign in to comment.