Skip to content

Commit 517d344

Browse files
sypharsyphar
committed
add download endpoint for rustdoc archive
1 parent dd66a73 commit 517d344

File tree

13 files changed

+407
-10
lines changed

13 files changed

+407
-10
lines changed

‎src/config.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ pub struct Config {
2626
#[cfg(test)]
2727
pub(crate) s3_bucket_is_temporary: bool,
2828

29+
// CloudFront domain which we can access
30+
// public S3 files through
31+
pub(crate) s3_static_domain: String,
32+
2933
// Github authentication
3034
pub(crate) github_accesstoken: Option<String>,
3135
pub(crate) github_updater_min_rate_limit: u32,
@@ -67,6 +71,8 @@ pub struct Config {
6771
// CloudFront distribution ID for the web server.
6872
// Will be used for invalidation-requests.
6973
pub cloudfront_distribution_id_web: Option<String>,
74+
/// same for the `static.docs.rs` distribution
75+
pub cloudfront_distribution_id_static: Option<String>,
7076

7177
// Build params
7278
pub(crate) build_attempts: u16,
@@ -125,6 +131,8 @@ impl Config {
125131
#[cfg(test)]
126132
s3_bucket_is_temporary: false,
127133

134+
s3_static_domain: env("S3_STATIC_DOMAIN", "https://static.docs.rs".to_string())?,
135+
128136
github_accesstoken: maybe_env("DOCSRS_GITHUB_ACCESSTOKEN")?,
129137
github_updater_min_rate_limit: env("DOCSRS_GITHUB_UPDATER_MIN_RATE_LIMIT", 2500)?,
130138

@@ -148,6 +156,7 @@ impl Config {
148156
cdn_backend: env("DOCSRS_CDN_BACKEND", CdnKind::Dummy)?,
149157

150158
cloudfront_distribution_id_web: maybe_env("CLOUDFRONT_DISTRIBUTION_ID_WEB")?,
159+
cloudfront_distribution_id_static: maybe_env("CLOUDFRONT_DISTRIBUTION_ID_STATIC")?,
151160

152161
local_archive_cache_path: env(
153162
"DOCSRS_ARCHIVE_INDEX_CACHE_PATH",

‎src/db/file.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,12 @@ pub fn add_path_into_remote_archive<P: AsRef<Path>>(
3838
storage: &Storage,
3939
archive_path: &str,
4040
path: P,
41+
public_access: bool,
4142
) -> Result<(Value, CompressionAlgorithm)> {
4243
let (file_list, algorithm) = storage.store_all_in_archive(archive_path, path.as_ref())?;
44+
if public_access {
45+
storage.set_public_access(archive_path, true)?;
46+
}
4347
Ok((
4448
file_list_to_json(file_list.into_iter().collect()),
4549
algorithm,

‎src/db/migrate.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,11 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> crate::error::Res
848848
"CREATE INDEX builds_release_id_idx ON builds (rid);",
849849
"DROP INDEX builds_release_id_idx;",
850850
),
851+
sql_migration!(
852+
context, 35, "add public visibility to files table",
853+
"ALTER TABLE files ADD COLUMN public BOOL NOT NULL DEFAULT FALSE;",
854+
"ALTER TABLE files DROP COLUMN public;"
855+
),
851856

852857
];
853858

‎src/docbuilder/rustwide_builder.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@ impl RustwideBuilder {
410410
&self.storage,
411411
&rustdoc_archive_path(name, version),
412412
local_storage.path(),
413+
true,
413414
)?;
414415
algs.insert(new_alg);
415416
};
@@ -421,6 +422,7 @@ impl RustwideBuilder {
421422
&self.storage,
422423
&source_archive_path(name, version),
423424
build.host_source_dir(),
425+
false,
424426
)?;
425427
algs.insert(new_alg);
426428
files_list

‎src/storage/database.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,32 @@ impl DatabaseBackend {
2121
Ok(conn.query(query, &[&path])?[0].get(0))
2222
}
2323

24+
pub(super) fn get_public_access(&self, path: &str) -> Result<bool> {
25+
match self.pool.get()?.query_opt(
26+
"SELECT public
27+
FROM files
28+
WHERE path = $1",
29+
&[&path],
30+
)? {
31+
Some(row) => Ok(row.get(0)),
32+
None => Err(super::PathNotFoundError.into()),
33+
}
34+
}
35+
36+
pub(super) fn set_public_access(&self, path: &str, public: bool) -> Result<()> {
37+
if self.pool.get()?.execute(
38+
"UPDATE files
39+
SET public = $2
40+
WHERE path = $1",
41+
&[&path, &public],
42+
)? == 1
43+
{
44+
Ok(())
45+
} else {
46+
Err(super::PathNotFoundError.into())
47+
}
48+
}
49+
2450
pub(super) fn get(
2551
&self,
2652
path: &str,

‎src/storage/mod.rs‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,20 @@ impl Storage {
140140
}
141141
}
142142

143+
pub(crate) fn get_public_access(&self, path: &str) -> Result<bool> {
144+
match &self.backend {
145+
StorageBackend::Database(db) => db.get_public_access(path),
146+
StorageBackend::S3(s3) => s3.get_public_access(path),
147+
}
148+
}
149+
150+
pub(crate) fn set_public_access(&self, path: &str, public: bool) -> Result<()> {
151+
match &self.backend {
152+
StorageBackend::Database(db) => db.set_public_access(path, public),
153+
StorageBackend::S3(s3) => s3.set_public_access(path, public),
154+
}
155+
}
156+
143157
fn max_file_size_for(&self, path: &str) -> usize {
144158
if path.ends_with(".html") {
145159
self.config.max_file_size_html
@@ -620,9 +634,38 @@ mod backend_tests {
620634
Ok(())
621635
}
622636

637+
fn test_set_public(storage: &Storage) -> Result<()> {
638+
let path: &str = "foo/bar.txt";
639+
640+
storage.store_blobs(vec![Blob {
641+
path: path.into(),
642+
mime: "text/plain".into(),
643+
date_updated: Utc::now(),
644+
compression: None,
645+
content: b"test content\n".to_vec(),
646+
}])?;
647+
648+
assert!(!storage.get_public_access(path)?);
649+
storage.set_public_access(path, true)?;
650+
assert!(storage.get_public_access(path)?);
651+
storage.set_public_access(path, false)?;
652+
assert!(!storage.get_public_access(path)?);
653+
654+
for path in &["bar.txt", "baz.txt", "foo/baz.txt"] {
655+
assert!(storage
656+
.set_public_access(path, true)
657+
.unwrap_err()
658+
.downcast_ref::<PathNotFoundError>()
659+
.is_some());
660+
}
661+
662+
Ok(())
663+
}
664+
623665
fn test_get_object(storage: &Storage) -> Result<()> {
666+
let path: &str = "foo/bar.txt";
624667
let blob = Blob {
625-
path: "foo/bar.txt".into(),
668+
path: path.into(),
626669
mime: "text/plain".into(),
627670
date_updated: Utc::now(),
628671
compression: None,
@@ -631,16 +674,25 @@ mod backend_tests {
631674

632675
storage.store_blobs(vec![blob.clone()])?;
633676

634-
let found = storage.get("foo/bar.txt", std::usize::MAX)?;
677+
let found = storage.get(path, std::usize::MAX)?;
635678
assert_eq!(blob.mime, found.mime);
636679
assert_eq!(blob.content, found.content);
637680

681+
// default visibility is private
682+
assert!(!storage.get_public_access(path)?);
683+
638684
for path in &["bar.txt", "baz.txt", "foo/baz.txt"] {
639685
assert!(storage
640686
.get(path, std::usize::MAX)
641687
.unwrap_err()
642688
.downcast_ref::<PathNotFoundError>()
643689
.is_some());
690+
691+
assert!(storage
692+
.get_public_access(path)
693+
.unwrap_err()
694+
.downcast_ref::<PathNotFoundError>()
695+
.is_some());
644696
}
645697

646698
Ok(())
@@ -1028,6 +1080,7 @@ mod backend_tests {
10281080
test_delete_prefix_without_matches,
10291081
test_delete_percent,
10301082
test_exists_without_remote_archive,
1083+
test_set_public,
10311084
}
10321085

10331086
tests_with_metrics {

‎src/storage/s3.rs‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use crate::{Config, Metrics};
33
use anyhow::{Context, Error};
44
use aws_sdk_s3::{
55
error,
6-
model::{Delete, ObjectIdentifier},
6+
model::{Delete, ObjectIdentifier, Tag, Tagging},
77
types::SdkError,
88
Client, Endpoint, Region, RetryConfig,
99
};
@@ -16,6 +16,9 @@ use futures_util::{
1616
use std::{io::Write, sync::Arc};
1717
use tokio::runtime::Runtime;
1818

19+
const PUBLIC_ACCESS_TAG: &str = "static-cloudfront-access";
20+
const PUBLIC_ACCESS_VALUE: &str = "allow";
21+
1922
pub(super) struct S3Backend {
2023
client: Client,
2124
runtime: Arc<Runtime>,
@@ -90,6 +93,71 @@ impl S3Backend {
9093
})
9194
}
9295

96+
pub(super) fn get_public_access(&self, path: &str) -> Result<bool, Error> {
97+
self.runtime.block_on(async {
98+
match self
99+
.client
100+
.get_object_tagging()
101+
.bucket(&self.bucket)
102+
.key(path)
103+
.send()
104+
.await
105+
{
106+
Ok(tags) => Ok(tags
107+
.tag_set()
108+
.map(|tags| {
109+
tags.iter()
110+
.filter(|tag| tag.key() == Some(PUBLIC_ACCESS_TAG))
111+
.any(|tag| tag.value() == Some(PUBLIC_ACCESS_VALUE))
112+
})
113+
.unwrap_or(false)),
114+
Err(SdkError::ServiceError { err, raw }) => {
115+
if raw.http().status() == http::StatusCode::NOT_FOUND {
116+
Err(super::PathNotFoundError.into())
117+
} else {
118+
Err(err.into())
119+
}
120+
}
121+
Err(other) => Err(other.into()),
122+
}
123+
})
124+
}
125+
126+
pub(super) fn set_public_access(&self, path: &str, public: bool) -> Result<(), Error> {
127+
self.runtime.block_on(async {
128+
match self
129+
.client
130+
.put_object_tagging()
131+
.bucket(&self.bucket)
132+
.key(path)
133+
.tagging(if public {
134+
Tagging::builder()
135+
.tag_set(
136+
Tag::builder()
137+
.key(PUBLIC_ACCESS_TAG)
138+
.value(PUBLIC_ACCESS_VALUE)
139+
.build(),
140+
)
141+
.build()
142+
} else {
143+
Tagging::builder().build()
144+
})
145+
.send()
146+
.await
147+
{
148+
Ok(_) => Ok(()),
149+
Err(SdkError::ServiceError { err, raw }) => {
150+
if raw.http().status() == http::StatusCode::NOT_FOUND {
151+
Err(super::PathNotFoundError.into())
152+
} else {
153+
Err(err.into())
154+
}
155+
}
156+
Err(other) => Err(other.into()),
157+
}
158+
})
159+
}
160+
93161
pub(super) fn get(
94162
&self,
95163
path: &str,

‎src/test/fakes.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -316,13 +316,21 @@ impl<'a> FakeRelease<'a> {
316316
source_directory.display()
317317
);
318318
if archive_storage {
319-
let archive = match kind {
320-
FileKind::Rustdoc => rustdoc_archive_path(&package.name, &package.version),
321-
FileKind::Sources => source_archive_path(&package.name, &package.version),
319+
let (archive, public) = match kind {
320+
FileKind::Rustdoc => {
321+
(rustdoc_archive_path(&package.name, &package.version), true)
322+
}
323+
FileKind::Sources => {
324+
(source_archive_path(&package.name, &package.version), false)
325+
}
322326
};
323327
log::debug!("store in archive: {:?}", archive);
324-
let (files_list, new_alg) =
325-
crate::db::add_path_into_remote_archive(&storage, &archive, source_directory)?;
328+
let (files_list, new_alg) = crate::db::add_path_into_remote_archive(
329+
&storage,
330+
&archive,
331+
source_directory,
332+
public,
333+
)?;
326334
let mut hm = HashSet::new();
327335
hm.insert(new_alg);
328336
Ok((files_list, hm))

‎src/web/routes.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ pub(super) fn build_routes() -> Routes {
101101
"/crate/:name/:version/builds",
102102
super::builds::build_list_handler,
103103
);
104+
routes.internal_page(
105+
"/crate/:name/:version/download",
106+
super::rustdoc::download_handler,
107+
);
104108
routes.static_resource(
105109
"/crate/:name/:version/builds.json",
106110
super::builds::build_list_handler,

0 commit comments

Comments
 (0)