Add update tool

Author: arlosi

.env.sample

@@ -33,6 +33,14 @@ export TEST_DATABASE_URL=
 # not needed if the S3 bucket is in US standard
 # export S3_REGION=
 
+# Credentials for uploading index metadata to S3. You can leave these commented
+# out if you're not publishing to s3 from your crates.io instance.
+# export S3_INDEX_BUCKET
+# export S3_INDEX_ACCESS_KEY
+# export S3_INDEX_SECRET_KEY
+# not needed if the S3 bucket is in US standard
+# export S3_INDEX_REGION
+
 # Upstream location of the registry index. Background jobs will push to
 # this URL. The default points to a local index for development.
 # Run `./script/init-local-index.sh` to initialize this repo.

cargo-registry-index/lib.rs

@@ -293,18 +293,33 @@ impl Repository {
             .join(Self::relative_index_file(name))
     }
 
+    /// Returns the relative path to the crate index file.
+    /// Does not perform conversion to lowercase.
+    fn relative_index_file_helper(name: &str) -> Vec<&str> {
+        match name.len() {
+            1 => vec!["1", &name],
+            2 => vec!["2", &name],
+            3 => vec!["3", &name[..1], &name],
+            _ => vec![&name[0..2], &name[2..4], &name],
+        }
+    }
+
     /// Returns the relative path to the crate index file that corresponds to
-    /// the given crate name.
+    /// the given crate name as a path.
     ///
     /// see <https://doc.rust-lang.org/cargo/reference/registries.html#index-format>
     pub fn relative_index_file(name: &str) -> PathBuf {
         let name = name.to_lowercase();
-        match name.len() {
-            1 => Path::new("1").join(&name),
-            2 => Path::new("2").join(&name),
-            3 => Path::new("3").join(&name[..1]).join(&name),
-            _ => Path::new(&name[0..2]).join(&name[2..4]).join(&name),
-        }
+        Self::relative_index_file_helper(&name).iter().collect()
+    }
+
+    /// Returns the relative path to the crate index file that corresponds to
+    /// the given crate name as a string.
+    ///
+    /// see <https://doc.rust-lang.org/cargo/reference/registries.html#index-format>
+    pub fn relative_index_file_as_string(name: &str) -> String {
+        let name = name.to_lowercase();
+        Self::relative_index_file_helper(&name).join("/")
     }
 
     /// Returns the [Object ID](git2::Oid) of the currently checked out commit
@@ -343,6 +358,32 @@ impl Repository {
         self.push("refs/heads/master")
     }
 
+    /// Gets a list of files that have been modified since a given commit (or None for all files).
+    pub fn get_files_modified_since(&self, starting_commit: Option<&str>) -> anyhow::Result<Vec<PathBuf>>{
+        let starting_commit = match starting_commit {
+            Some(starting_commit) => {
+                let oid = git2::Oid::from_str(starting_commit).context("failed to parse commit into Oid")?;
+                let commit = self.repository.find_commit(oid).context("failed to find commit")?;
+                Some(commit.as_object().peel_to_tree().context("failed to find tree for commit")?)
+            }
+            None => None,
+        };
+
+        let head = self.repository.find_commit(self.head_oid()?)?.as_object().peel_to_tree()?;
+        let diff = self.repository
+            .diff_tree_to_tree(starting_commit.as_ref(), Some(&head), None)
+            .context("failed to run diff")?;
+        let mut files = Vec::new();
+        for delta in diff.deltas() {
+            let file = delta.new_file();
+            if file.exists() {
+                files.push(file.path().unwrap().to_path_buf());
+            }
+        }
+
+        Ok(files)
+    }
+
     /// Push the current branch to the provided refname
     fn push(&self, refspec: &str) -> anyhow::Result<()> {
         let mut ref_status = Ok(());

src/admin/mod.rs

@@ -8,3 +8,4 @@ pub mod render_readmes;
 pub mod test_pagerduty;
 pub mod transfer_crates;
 pub mod verify_token;
+pub mod upload_index;

src/admin/render_readmes.rs

@@ -14,7 +14,6 @@ use flate2::read::GzDecoder;
 use reqwest::{blocking::Client, header};
 use tar::{self, Archive};
 
-const CACHE_CONTROL_README: &str = "public,max-age=604800";
 const USER_AGENT: &str = "crates-admin";
 
 #[derive(clap::Parser, Debug)]
@@ -111,24 +110,14 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
             let handle = thread::spawn::<_, anyhow::Result<()>>(move || {
                 println!("[{}-{}] Rendering README...", krate_name, version.num);
                 let readme = get_readme(base_config.uploader(), &client, &version, &krate_name)?;
-                let content_length = readme.len() as u64;
-                let content = std::io::Cursor::new(readme);
-                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
-                let mut extra_headers = header::HeaderMap::new();
-                extra_headers.insert(
-                    header::CACHE_CONTROL,
-                    header::HeaderValue::from_static(CACHE_CONTROL_README),
-                );
 
                 base_config
                     .uploader()
-                    .upload(
+                    .upload_readme(
                         &client,
-                        &readme_path,
-                        content,
-                        content_length,
-                        "text/html",
-                        extra_headers,
+                        &krate_name,
+                        &version.num,
+                        readme,
                     )
                     .context("Failed to upload rendered README file to S3")?;
 

src/bin/crates-admin.rs

@@ -2,7 +2,7 @@
 
 use cargo_registry::admin::{
     delete_crate, delete_version, migrate, populate, render_readmes, test_pagerduty,
-    transfer_crates, verify_token,
+    transfer_crates, verify_token, upload_index,
 };
 
 #[derive(clap::Parser, Debug)]
@@ -22,6 +22,7 @@ enum SubCommand {
     TransferCrates(transfer_crates::Opts),
     VerifyToken(verify_token::Opts),
     Migrate(migrate::Opts),
+    UploadIndex(upload_index::Opts),
 }
 
 fn main() -> anyhow::Result<()> {
@@ -38,6 +39,7 @@ fn main() -> anyhow::Result<()> {
         SubCommand::TransferCrates(opts) => transfer_crates::run(opts),
         SubCommand::VerifyToken(opts) => verify_token::run(opts).unwrap(),
         SubCommand::Migrate(opts) => migrate::run(opts)?,
+        SubCommand::UploadIndex(opts) => upload_index::run(opts)?,
     }
 
     Ok(())

src/config/base.rs

@@ -83,6 +83,15 @@ impl Base {
                 // sniff/record it, but everywhere else we use https
                 "http",
             ),
+            index_bucket: s3::Bucket::new(
+                String::from("alexcrichton-test"),
+                None,
+                dotenv::var("S3_INDEX_ACCESS_KEY").unwrap_or_default(),
+                dotenv::var("S3_INDEX_SECRET_KEY").unwrap_or_default(),
+                // When testing we route all API traffic over HTTP so we can
+                // sniff/record it, but everywhere else we use https
+                "http",
+            ),
             cdn: None,
         };
         Self {
@@ -104,6 +113,13 @@ impl Base {
                 env("S3_SECRET_KEY"),
                 "https",
             ),
+            index_bucket: s3::Bucket::new(
+                env("S3_INDEX_BUCKET"),
+                dotenv::var("S3_INDEX_REGION").ok(),
+                env("S3_INDEX_ACCESS_KEY"),
+                env("S3_INDEX_SECRET_KEY"),
+                "https",
+            ),
             cdn: dotenv::var("S3_CDN").ok(),
         }
     }
@@ -117,6 +133,13 @@ impl Base {
                 dotenv::var("S3_SECRET_KEY").unwrap_or_default(),
                 "https",
             ),
+            index_bucket: s3::Bucket::new(
+                env("S3_INDEX_BUCKET"),
+                dotenv::var("S3_INDEX_REGION").ok(),
+                dotenv::var("S3_INDEX_ACCESS_KEY").unwrap_or_default(),
+                dotenv::var("S3_INDEX_SECRET_KEY").unwrap_or_default(),
+                "https",
+            ),
             cdn: dotenv::var("S3_CDN").ok(),
         }
     }

src/uploaders.rs

@@ -21,6 +21,7 @@ pub enum Uploader {
     /// For test usage with `TestApp::with_proxy()`, the recording proxy is used.
     S3 {
         bucket: s3::Bucket,
+        index_bucket: s3::Bucket,
         cdn: Option<String>,
     },
 
@@ -29,6 +30,11 @@ pub enum Uploader {
     Local,
 }
 
+pub enum UploadType {
+    Default,
+    Index,
+}
+
 impl Uploader {
     /// Returns the URL of an uploaded crate's version archive.
     ///
@@ -74,7 +80,6 @@ impl Uploader {
 
     /// Returns the internal path of an uploaded crate's version archive.
     fn crate_path(name: &str, version: &str) -> String {
-        // No slash in front so we can use join
         format!("crates/{name}/{name}-{version}.crate")
     }
 
@@ -85,13 +90,8 @@ impl Uploader {
 
     /// Returns the internal path of an uploaded crate's index file.
     fn index_path(name: &str) -> String {
-        let prefix = match name.len() {
-            1 => String::from("1"),
-            2 => String::from("2"),
-            3 => format!("3/{}", &name[..1]),
-            _ => format!("{}/{}", &name[0..2], &name[2..4]),
-        };
-        format!("index/{prefix}/{name}")
+        let path = cargo_registry_index::Repository::relative_index_file_as_string(name);
+        format!("index/{}", path)
     }
 
     /// Uploads a file using the configured uploader (either `S3`, `Local`).
@@ -110,9 +110,14 @@ impl Uploader {
         content_length: u64,
         content_type: &str,
         extra_headers: header::HeaderMap,
+        upload_type: UploadType,
     ) -> Result<Option<String>> {
         match *self {
-            Uploader::S3 { ref bucket, .. } => {
+            Uploader::S3 { ref bucket, ref index_bucket, .. } => {
+                let bucket = match upload_type {
+                    UploadType::Index => index_bucket,
+                    UploadType::Default => bucket,
+                };
                 bucket.put(
                     client,
                     path,
@@ -157,6 +162,7 @@ impl Uploader {
             content_length,
             "application/x-tar",
             extra_headers,
+            UploadType::Default,
         )
         .map_err(|e| internal(&format_args!("failed to upload crate: {}", e)))?;
         Ok(())
@@ -184,6 +190,7 @@ impl Uploader {
             content_length,
             "text/html",
             extra_headers,
+            UploadType::Default,
         )?;
         Ok(())
     }
@@ -209,6 +216,7 @@ impl Uploader {
             content_length,
             "text/plain",
             extra_headers,
+            UploadType::Index,
         )?;
         Ok(())
     }

src/worker/dump_db.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use self::configuration::VisibilityConfig;
-use crate::{background_jobs::Environment, uploaders::Uploader};
+use crate::{background_jobs::Environment, uploaders::{Uploader, UploadType}};
 use reqwest::header;
 use swirl::PerformError;
 
@@ -200,6 +200,7 @@ impl DumpTarball {
             content_length,
             "application/gzip",
             header::HeaderMap::new(),
+            UploadType::Default,
         )?;
         Ok(content_length)
     }

src/worker/git.rs

@@ -9,7 +9,7 @@ use std::process::Command;
 use swirl::PerformError;
 
 #[swirl::background_job]
-pub fn add_crate(env: &Environment, krate: Crate) -> Result<(), PerformError> {
+pub fn add_crate(env: &Environment, conn: &PgConnection, krate: Crate) -> Result<(), PerformError> {
     use std::io::prelude::*;
 
     let repo = env.lock_index()?;
@@ -20,20 +20,25 @@ pub fn add_crate(env: &Environment, krate: Crate) -> Result<(), PerformError> {
     let mut file = OpenOptions::new()
         .append(true)
         .create(true)
-        .read(true)
         .open(&dst)?;
     serde_json::to_writer(&mut file, &krate)?;
     file.write_all(b"\n")?;
 
-    file.rewind()?;
-    let mut contents = String::new();
-    file.read_to_string(&mut contents)?;
-    env.uploader
-        .upload_index(env.http_client(), &krate.name, contents)?;
-
     let message: String = format!("Updating crate `{}#{}`", krate.name, krate.vers);
     repo.commit_and_push(&message, &dst)?;
 
+    // Queue another background job to update the http-based index as well.
+    update_crate_index(krate.name.clone()).enqueue(&conn)?;
+    Ok(())
+}
+
+#[swirl::background_job]
+pub fn update_crate_index(env: &Environment, crate_name: String) -> Result<(), PerformError> {
+    let repo = env.lock_index()?;
+    let dst = repo.index_file(&crate_name);
+    let contents = std::fs::read_to_string(dst)?;
+    env.uploader
+        .upload_index(env.http_client(), &crate_name, contents)?;
     Ok(())
 }