Skip to content

Commit 26c57f1

Browse files
davidanthoffdavidanthoff
committed
Update package indexing
1 parent 586e161 commit 26c57f1

File tree

4 files changed

+148
-69
lines changed

4 files changed

+148
-69
lines changed

‎registryindexer/Manifest.toml‎

Lines changed: 110 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,36 @@
11
# This file is machine-generated - editing it directly is not advised
22

3+
[[ArgTools]]
4+
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
5+
6+
[[Artifacts]]
7+
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
8+
39
[[Base64]]
410
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
511

12+
[[CodecZlib]]
13+
deps = ["TranscodingStreams", "Zlib_jll"]
14+
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
15+
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
16+
version = "0.7.0"
17+
18+
[[Compat]]
19+
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
20+
git-tree-sha1 = "dc7dedc2c2aa9faf59a55c622760a25cbefbe941"
21+
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
22+
version = "3.31.0"
23+
624
[[DataAPI]]
7-
git-tree-sha1 = "ad84f52c0b8f05aa20839484dbaf01690b41ff84"
25+
git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
826
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
9-
version = "1.4.0"
27+
version = "1.7.0"
1028

1129
[[DataStructures]]
12-
deps = ["InteractiveUtils", "OrderedCollections"]
13-
git-tree-sha1 = "88d48e133e6d3dd68183309877eac74393daa7eb"
30+
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
31+
git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
1432
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
15-
version = "0.17.20"
33+
version = "0.18.9"
1634

1735
[[DataValueInterfaces]]
1836
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
@@ -29,10 +47,18 @@ version = "0.4.13"
2947
deps = ["Printf"]
3048
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
3149

50+
[[DelimitedFiles]]
51+
deps = ["Mmap"]
52+
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
53+
3254
[[Distributed]]
3355
deps = ["Random", "Serialization", "Sockets"]
3456
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
3557

58+
[[Downloads]]
59+
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
60+
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
61+
3662
[[InteractiveUtils]]
3763
deps = ["Markdown"]
3864
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -54,6 +80,22 @@ git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
5480
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
5581
version = "0.21.1"
5682

83+
[[LibCURL]]
84+
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
85+
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
86+
87+
[[LibCURL_jll]]
88+
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
89+
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
90+
91+
[[LibGit2]]
92+
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
93+
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
94+
95+
[[LibSSH2_jll]]
96+
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
97+
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
98+
5799
[[Libdl]]
58100
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
59101

@@ -74,35 +116,49 @@ version = "0.5.6"
74116
deps = ["Base64"]
75117
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
76118

119+
[[MbedTLS_jll]]
120+
deps = ["Artifacts", "Libdl"]
121+
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
122+
77123
[[Missings]]
78124
deps = ["DataAPI"]
79-
git-tree-sha1 = "ed61674a0864832495ffe0a7e889c0da76b0f4c8"
125+
git-tree-sha1 = "f8c673ccc215eb50fcadb285f522420e29e69e1c"
80126
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
81-
version = "0.4.4"
127+
version = "0.4.5"
82128

83129
[[Mmap]]
84130
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
85131

132+
[[MozillaCACerts_jll]]
133+
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
134+
135+
[[NetworkOptions]]
136+
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
137+
86138
[[OrderedCollections]]
87-
git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db"
139+
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
88140
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
89-
version = "1.3.2"
141+
version = "1.4.1"
90142

91143
[[Parsers]]
92144
deps = ["Dates"]
93-
git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714"
145+
git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
94146
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
95-
version = "1.0.15"
147+
version = "1.1.0"
148+
149+
[[Pkg]]
150+
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
151+
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
96152

97153
[[Printf]]
98154
deps = ["Unicode"]
99155
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
100156

101157
[[ProgressMeter]]
102158
deps = ["Distributed", "Printf"]
103-
git-tree-sha1 = "45640774ee2efa24e52686dbdf895e88102e68fc"
159+
git-tree-sha1 = "afadeba63d90ff223a6a48d2009434ecee2ec9e8"
104160
uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
105-
version = "1.4.1"
161+
version = "1.7.1"
106162

107163
[[Query]]
108164
deps = ["DataValues", "IterableTables", "MacroTools", "QueryOperators", "Statistics"]
@@ -112,26 +168,34 @@ version = "1.0.0"
112168

113169
[[QueryOperators]]
114170
deps = ["DataStructures", "DataValues", "IteratorInterfaceExtensions", "TableShowUtils"]
115-
git-tree-sha1 = "c233ed156b7628ea5a730c3b856b471a90b58afd"
171+
git-tree-sha1 = "911c64c204e7ecabfd1872eb93c49b4e7c701f02"
116172
uuid = "2aef5ad7-51ca-5a8f-8e88-e75cf067b44b"
117-
version = "0.9.2"
173+
version = "0.9.3"
174+
175+
[[REPL]]
176+
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
177+
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
118178

119179
[[Random]]
120180
deps = ["Serialization"]
121181
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
122182

123183
[[Requires]]
124184
deps = ["UUIDs"]
125-
git-tree-sha1 = "cfbac6c1ed70c002ec6361e7fd334f02820d6419"
185+
git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
126186
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
127-
version = "1.1.2"
187+
version = "1.1.3"
128188

129189
[[SHA]]
130190
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
131191

132192
[[Serialization]]
133193
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
134194

195+
[[SharedArrays]]
196+
deps = ["Distributed", "Mmap", "Random", "Serialization"]
197+
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
198+
135199
[[Sockets]]
136200
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
137201

@@ -143,6 +207,10 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
143207
deps = ["LinearAlgebra", "SparseArrays"]
144208
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
145209

210+
[[TOML]]
211+
deps = ["Dates"]
212+
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
213+
146214
[[TableShowUtils]]
147215
deps = ["DataValues", "Dates", "JSON", "Markdown", "Test"]
148216
git-tree-sha1 = "14c54e1e96431fb87f0d2f5983f090f1b9d06457"
@@ -151,23 +219,45 @@ version = "0.2.5"
151219

152220
[[TableTraits]]
153221
deps = ["IteratorInterfaceExtensions"]
154-
git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e"
222+
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
155223
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
156-
version = "1.0.0"
224+
version = "1.0.1"
157225

158226
[[TableTraitsUtils]]
159227
deps = ["DataValues", "IteratorInterfaceExtensions", "Missings", "TableTraits"]
160228
git-tree-sha1 = "8fc12ae66deac83e44454e61b02c37b326493233"
161229
uuid = "382cd787-c1b6-5bf2-a167-d5b971a19bda"
162230
version = "1.0.1"
163231

232+
[[Tar]]
233+
deps = ["ArgTools", "SHA"]
234+
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
235+
164236
[[Test]]
165-
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
237+
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
166238
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
167239

240+
[[TranscodingStreams]]
241+
deps = ["Random", "Test"]
242+
git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
243+
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
244+
version = "0.9.5"
245+
168246
[[UUIDs]]
169247
deps = ["Random", "SHA"]
170248
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
171249

172250
[[Unicode]]
173251
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
252+
253+
[[Zlib_jll]]
254+
deps = ["Libdl"]
255+
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
256+
257+
[[nghttp2_jll]]
258+
deps = ["Artifacts", "Libdl"]
259+
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
260+
261+
[[p7zip_jll]]
262+
deps = ["Artifacts", "Libdl"]
263+
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"

‎registryindexer/Project.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
[deps]
2+
CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
23
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
34
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
45
Query = "1a8c2f83-1ff3-5112-b086-8aa67b057ba1"
6+
Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

‎registryindexer/main.jl‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,15 @@ max_n = 1_000_000
44
max_versions = 1_000_000
55
max_tasks = length(ARGS)>1 ? parse(Int, ARGS[2]) : 1
66

7-
julia_versions = [v"1.5.3"]
7+
julia_versions = [v"1.6.1"]
88

9-
using Pkg, UUIDs
9+
using Pkg, UUIDs, Tar,
1010

1111
Pkg.activate(@__DIR__)
1212
Pkg.instantiate()
1313

1414
using ProgressMeter, Query, JSON
1515

16-
Pkg.PlatformEngines.probe_platform_engines!()
17-
1816
function get_all_package_versions(;max_versions=typemax(Int))
1917
registry_folder_path = joinpath(homedir(), ".julia", "registries", "General")
2018
registry_path = joinpath(registry_folder_path, "Registry.toml")
@@ -185,25 +183,25 @@ asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
185183
cache_path = joinpath(cache_folder, "v1", "packages", string(uppercase(v.name[1])), "$(v.name)_$(v.uuid)")
186184
cache_path_compressed = joinpath(cache_path, "v$(versionwithoutplus)_$(v.treehash).tar.gz")
187185

188-
res = execute(`docker run --rm --mount type=bind,source="$cache_folder",target=/symcache juliavscodesymbolindexer:$(first(julia_versions)) julia SymbolServer/src/indexpackage.jl $(v.name) $(v.version) $(v.uuid) $(v.treehash)`)
186+
mktempdir() do path
187+
res = execute(`docker run --rm --mount type=bind,source="$path",target=/symcache juliavscodesymbolindexer:$(first(julia_versions)) julia SymbolServer/src/indexpackage.jl $(v.name) $(v.version) $(v.uuid) $(v.treehash)`)
189188

190-
if res.code==37 # This is our magic error code that indicates everything worked
191-
global count_successfully_cached += 1
192-
else
193-
if res.code==10
194-
global count_failed_to_load += 1
195-
elseif res.code==20
196-
global count_failed_to_install += 1
189+
if res.code==37 # This is our magic error code that indicates everything worked
190+
global count_successfully_cached += 1
197191
else
198-
global count_failed_to_index += 1
199-
end
192+
if res.code==10
193+
global count_failed_to_load += 1
194+
elseif res.code==20
195+
global count_failed_to_install += 1
196+
else
197+
global count_failed_to_index += 1
198+
end
200199

201-
@info res.code
200+
@info res.code
202201

203-
@info res.stdout
204-
@info res.stderr
202+
@info res.stdout
203+
@info res.stderr
205204

206-
mktempdir() do path
207205
error_filename = "v$(versionwithoutplus)_$(v.treehash).unavailable"
208206

209207
isfile(joinpath(path, error_filename)) && rm(joinpath(path, error_filename))
@@ -212,27 +210,29 @@ asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
212210
open(joinpath(path, error_filename), "w") do io
213211
end
214212

215-
@info "Files to be compressed" path error_filename readdir(path, join=true) ispath(cache_path) isfile(cache_path_compressed)
216-
217-
# Pkg.PlatformEngines.package(path, cache_path_compressed)
213+
open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stdout.txt"), "w") do f
214+
print(f, res.stdout)
215+
end
218216

219-
withenv("GZIP" => "-9") do
220-
cmd = Pkg.PlatformEngines.gen_package_cmd(path, cache_path_compressed)
221-
run(cmd)
217+
open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stderr.txt"), "w") do f
218+
print(f, res.stderr)
222219
end
223-
end
224220

225-
open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stdout.txt"), "w") do f
226-
print(f, res.stdout)
227-
end
221+
global status_db
228222

229-
open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stderr.txt"), "w") do f
230-
print(f, res.stderr)
223+
push!(status_db, Dict("name"=>v.name, "uuid"=>string(v.uuid), "version"=>string(v.version), "treehash"=>v.treehash, "status"=>res.code==20 ? "install_error" : res.code==10 ? "load_error" : "index_error", "indexattempts"=>[Dict("juliaversion"=>string(VERSION), "stdout"=>res.stdout, "stderr"=>res.stderr)]))
231224
end
232225

233-
global status_db
226+
@info "Files to be compressed" path readdir(path, join=true) ispath(cache_path) isfile(cache_path_compressed)
234227

235-
push!(status_db, Dict("name"=>v.name, "uuid"=>string(v.uuid), "version"=>string(v.version), "treehash"=>v.treehash, "status"=>res.code==20 ? "install_error" : res.code==10 ? "load_error" : "index_error", "indexattempts"=>[Dict("juliaversion"=>string(VERSION), "stdout"=>res.stdout, "stderr"=>res.stderr)]))
228+
open(cache_path_compressed, write=true) do tar_gz
229+
tar = GzipCompressorStream(tar_gz)
230+
try
231+
Tar.create(path, tar)
232+
finally
233+
close(tar)
234+
end
235+
end
236236
end
237237

238238
next!(p, showvalues = [

‎src/indexpackage.jl‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,7 @@ current_package_treehash = ARGS[4]
1414
store_path = "/symcache"
1515

1616
current_package_versionwithoutplus = replace(string(current_package_version), '+'=>'_')
17-
cache_package_folder_path = joinpath(store_path, "v1", "packages", string(uppercase(string(current_package_name)[1])), "$(current_package_name)_$current_package_uuid")
18-
filename_without_extension = "v$(current_package_versionwithoutplus)_$current_package_treehash"
19-
filename_with_extension = "$filename_without_extension.jstore"
20-
cache_path = joinpath(cache_package_folder_path, filename_with_extension)
21-
cache_path_compressed = joinpath(cache_package_folder_path, "$filename_without_extension.tar.gz")
22-
23-
mkpath(cache_package_folder_path)
17+
filename_with_extension = "v$(current_package_versionwithoutplus)_$current_package_treehash.jstore"
2418

2519
module LoadingBay end
2620

@@ -56,16 +50,9 @@ modify_dirs(env[current_package_name], f -> modify_dir(f, pkg_src_dir(Base.loade
5650

5751
# There's an issue here - @enum used within CSTParser seems to add a method that is introduced from Enums.jl...
5852

59-
Pkg.PlatformEngines.probe_platform_engines!()
60-
61-
mktempdir() do path
62-
# Write them to a file
63-
open(joinpath(path, filename_with_extension), "w") do io
64-
CacheStore.write(io, Package(string(current_package_name), env[current_package_name], current_package_uuid, nothing))
65-
end
66-
67-
# cp(joinpath(path, filename_with_extension), cache_path)
68-
Pkg.PlatformEngines.package(path, cache_path_compressed)
53+
# Write them to a file
54+
open(joinpath(store_path, filename_with_extension), "w") do io
55+
CacheStore.write(io, Package(string(current_package_name), env[current_package_name], current_package_uuid, nothing))
6956
end
7057

7158
@info "Finished indexing."

0 commit comments

Comments
 (0)