Merge pull request #14 from open-rust-initiative/fix-delete-flag

Fix delete flag

src/cloud/s3.rs

@@ -1,11 +1,13 @@
-use log::info;
+use std::path::Path;
+
+use log::debug;
 
 use crate::errors::{FreightResult, FreighterError};
 
 /// provide a common file upload interface
 pub trait CloudStorage {
     /// upload a single file to target storage
-    // fn upload(&self, bucket: &str) -> FreightResult;
+    fn upload_file(&self, file_path: &Path, s3_path: &str, bucket: &str) -> FreightResult;
 
     /// this operation will upload all files in folder
     fn upload_folder(&self, folder: &str, bucket: &str) -> FreightResult;
@@ -16,7 +18,7 @@ pub struct S3cmd {}
 
 impl CloudStorage for S3cmd {
     fn upload_folder(&self, folder: &str, bucket: &str) -> FreightResult {
-        info!("trying to upload folder {} to s3", folder);
+        debug!("trying to upload folder {} to s3", folder);
         let status = std::process::Command::new("s3cmd")
             .arg("sync")
             .arg(folder)
@@ -29,4 +31,29 @@ impl CloudStorage for S3cmd {
         }
         Ok(())
     }
+
+    fn upload_file(&self, file_path: &Path, s3_path: &str, bucket: &str) -> FreightResult {
+        // cargo download url is https://crates.rust-lang.pub/crates/{name}/{version}/download
+        //
+
+        // Upload to the Digital Ocean Spaces with s3cmd
+        // URL: s3://rust-lang/crates/{}/{}
+        // cmd: s3cmd put {file_path} s3://rust-lang/crates/{s3_path} --acl-public
+        // cmd: s3cmd put {file_path} s3://rust-lang/crates/{s3_path} --acl-public --no-mime-magic
+        // cmd: s3cmd put {file_path} s3://rust-lang/crates/{s3_path} --acl-public --no-mime-magic --guess-mime-type
+        // cmd: s3cmd put {file_path} s3://rust-lang/crates/{s3_path} --acl-public --no-mime-magic --guess-mime-type --add-header="Content-Type: application/octet-stream"
+        let s3_full_path = format!("s3://{}/{}", bucket, s3_path);
+        debug!("s3_full_path: {}", s3_full_path);
+        let status = std::process::Command::new("s3cmd")
+            .arg("put")
+            .arg(file_path)
+            .arg(s3_full_path)
+            .arg("--acl-public")
+            .status()
+            .expect("failed to execute process");
+        if !status.success() {
+            return Err(FreighterError::code(status.code().unwrap()));
+        }
+        Ok(())
+    }
 }

src/commands/channel.rs

@@ -6,10 +6,10 @@
 //!
 //!   Arguments:
 //!   - __domain__: you can choose your own upstream by adding this arugment in command
-//!   - __download-threads__: specify the download threads to parallel download, 
+//!   - __download-threads__: specify the download threads to parallel download,
 //!        this param can be changed in the configuration file or pass it here
 //!   - __no-progressbar__: not implemented
-//! 
+//!
 //! # download subcommand
 //!   - before each download, freighter will try to fetch the sha256 of the file and compare with local file if it exists
 //!         and will skip downloading if they are matching.
@@ -23,9 +23,9 @@
 //!
 //!   Arguments:
 //!   - __clean__: clean history files read by config file after download successfully.
-//!   - __version__: only download the version you specified, 
+//!   - __version__: only download the version you specified,
 //!         you can provide any version format supported by rust-org, such as stable, beta or nightly-2022-07-31.
-//! 
+//!
 //! # upload subcommand
 //!   upload file to Object Storage Service compatible with [AWS S3](https://aws.amazon.com/s3/)
 //!     - Digitalocean Spaces
@@ -35,15 +35,15 @@
 //!     - AWS S3
 //!     - minio
 //!     - Ceph
-//! 
+//!
 //!   Arguments:
 //!   - __bucket__: set the s3 bucket you want to upload files to, you must provide this param befor uplaod.
 //!   
 
 use clap::{arg, ArgMatches};
 use log::info;
 
-use crate::cloud::s3::{S3cmd, CloudStorage};
+use crate::cloud::s3::{CloudStorage, S3cmd};
 use crate::commands::command_prelude::*;
 use crate::config::Config;
 use crate::crates::channel::{sync_rust_toolchain, ChannelOptions};
@@ -54,6 +54,9 @@ pub fn cli() -> clap::Command {
         .subcommand(subcommand("download")
             .arg(flag("clean", "clean up historical versions"))
             .arg(arg!(-v --"version" <VALUE> "only download the version you specified"))
+            .arg(flag("upload", "upload every crate file after download"))
+            .arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files"))
+            .arg(flag("delete-after-upload", "this will delete file after upload"))
         )
         .subcommand(subcommand("upload")
         .arg(
@@ -122,15 +125,28 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
     info!("ChannelOptions info : {:#?}", opts);
 
     match args.subcommand() {
-        Some(("download", args)) => sync_rust_toolchain(&ChannelOptions {
-            clean: args.get_flag("clean"),
-            version: args.get_one::<String>("version").cloned(),
-            ..opts
-        })?,
+        Some(("download", args)) => {
+            let bucket_name = args.get_one::<String>("bucket").cloned();
+            let upload = args.get_flag("upload");
+            if upload && bucket_name.is_none() {
+                unreachable!("can not upload with empty bucket name")
+            }
+
+            sync_rust_toolchain(&ChannelOptions {
+                clean: args.get_flag("clean"),
+                version: args.get_one::<String>("version").cloned(),
+                upload: upload,
+                delete_after_upload: args.get_flag("delete-after-upload"),
+                bucket_name: bucket_name.unwrap(),
+                ..opts
+            })?
+        },
         Some(("upload", args)) => {
             let bucket_name = args.get_one::<String>("bucket").cloned().unwrap();
             let s3cmd = S3cmd::default();
-            s3cmd.upload_folder(opts.dist_path.to_str().unwrap(), &bucket_name).unwrap();
+            s3cmd
+                .upload_folder(opts.dist_path.to_str().unwrap(), &bucket_name)
+                .unwrap();
         }
         Some((cmd, _)) => {
             unreachable!("unexpected command {}", cmd)

src/commands/crates.rs

@@ -11,9 +11,9 @@
 //!   - __no-progressbar__: Whether to hide progress bar when start downloading
 //!
 //! # pull subcommand
-//! 
+//!
 //!   sync crates index from upstream to local:
-//! 
+//!
 //!   - The crates index is a git repository, and **cargo** clone and update from [GitHub](https://github.com/rust-lang/crates.io-index).
 //!     - The clone use `bare` mode, more details in the [cargo guide](https://github.com/rust-lang/cargo/blob/6b6b0b486d73c03ed952591d880debec1d47c534/src/doc/src/guide/cargo-home.md#directories)
 //!   
@@ -28,10 +28,12 @@
 //!         URL_s3_primary: "https://crates-io.s3-us-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
 //!         URL_s3_fallback: "https://crates-io-fallback.s3-eu-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
 //!      ```
-//! 
+//!
 //!   Arguments:
 //!   - __init__: Whether to download all the crates files for initialization.
 //!   - __upload__: Whether to upload single file to s3 after download success.
+//!   - __bucket__: set the s3 bucket you want to upload files to, you must provide this param befor uplaod.
+//!   - __delete-after-upload__: This optional parameter will be used to delete files after upload.
 //!
 //! # upload subcommand
 //!
@@ -48,7 +50,7 @@
 //!  
 
 use clap::{arg, ArgMatches};
-use log::info;
+use log::{debug, info};
 
 use crate::commands::command_prelude::*;
 use crate::config::Config;
@@ -75,6 +77,8 @@ pub fn cli() -> clap::Command {
         .subcommand(subcommand("download")
             .arg(flag("init", "Start init download of crates file, this will traverse all index for full download"))
             .arg(flag("upload", "upload every crate file after download"))
+            .arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files"))
+            .arg(flag("delete-after-upload", "this will delete file after upload"))
         )
         .subcommand_required(true)
         .arg_required_else_help(true)
@@ -142,7 +146,13 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
         Some(("download", args)) => {
             opts.upload = args.get_flag("upload");
             opts.init_download = args.get_flag("init");
-
+            opts.delete_after_upload = args.get_flag("delete-after-upload");
+            let bucket_name = args.get_one::<String>("bucket").cloned();
+            if opts.upload && bucket_name.is_none() {
+                unreachable!("can not upload with empty bucket name")
+            }
+            opts.bucket_name = bucket_name.unwrap();
+            debug!("download command opts: {:?}", opts);
             if let Some(source) = domain {
                 config.crates.domain = source;
             }

src/config.default.toml

@@ -22,11 +22,8 @@ domain = "https://static.crates.io/crates"
 # Number of download threads
 download_threads = 2
 
-# when local file does not exist, redirect client to this domain
-redirect_domain = "https://rsproxy.cn"
-
 # when local file does not exist, will try to download from these doamins
-backup_domain = ["https://rsproxy.cn", "https://static.crates.io"]
+backup_domain = ["localhost", "https://rsproxy.cn", "https://static.crates.io"]
 
 [rustup]
 # download rustup from domain
@@ -134,11 +131,7 @@ sync_nightly_days = 30
 # days you want to keep for historical beta version
 sync_beta_days = 30
 
-
-# when local file does not exist, redirect client to this domain
-redirect_domain = "https://static.rust-lang.org"
-
 # when local file does not exist, will try to download from these doamins
-backup_domain = ["https://rsproxy.cn", "https://static.rust-lang.org"]
+backup_domain = ["localhost", "https://rsproxy.cn", "https://static.rust-lang.org"]
 
 

src/config.rs

@@ -32,7 +32,6 @@ pub struct CratesConfig {
     pub index_domain: String,
     pub domain: String,
     pub download_threads: usize,
-    pub redirect_domain: Option<String>,
     pub backup_domain: Option<Vec<String>>,
 }
 
@@ -44,7 +43,6 @@ pub struct RustUpConfig {
     pub sync_stable_versions: Vec<String>,
     pub sync_nightly_days: i64,
     pub sync_beta_days: i64,
-    pub redirect_domain: Option<String>,
     pub backup_domain: Option<Vec<String>>,
 
 }

src/crates/channel.rs

@@ -5,18 +5,18 @@ use std::{
 };
 
 use chrono::{Duration, NaiveDate, Utc};
-use log::{info, error};
+use log::{error, info};
 use serde::Deserialize;
 use threadpool::ThreadPool;
 use walkdir::WalkDir;
 
 use crate::{
+    cloud::s3::{CloudStorage, S3cmd},
     config::RustUpConfig,
     download::{download_file, download_file_with_sha},
     errors::{FreightResult, FreighterError},
 };
 
-
 #[derive(Debug, Deserialize)]
 pub struct Channel {
     #[serde(alias = "manifest-version")]
@@ -56,6 +56,10 @@ pub struct ChannelOptions {
     pub dist_path: PathBuf,
 
     pub bucket_name: String,
+
+    pub upload: bool,
+
+    pub delete_after_upload: bool,
 }
 
 /// entrance function
@@ -87,8 +91,6 @@ pub fn sync_rust_toolchain(opts: &ChannelOptions) -> FreightResult {
     Ok(())
 }
 
-
-
 // sync the latest toolchain by given a channel name(stable, beta, nightly) or history verison by version number
 pub fn sync_channel(opts: &ChannelOptions, channel: &str) -> FreightResult {
     let channel_name;
@@ -116,8 +118,22 @@ pub fn sync_channel(opts: &ChannelOptions, channel: &str) -> FreightResult {
                         url.split('/').map(PathBuf::from).collect::<Vec<PathBuf>>()[4..].to_owned(),
                     )
                     .collect();
+                let (upload, dist_path, bucket_name, delete_after_upload) = (
+                    opts.upload,
+                    opts.dist_path.to_owned(),
+                    opts.bucket_name.to_owned(),
+                    opts.delete_after_upload,
+                );
+                let s3cmd = S3cmd::default();
                 pool.execute(move || {
-                    download_file(&url, &path, Some(&hash), false).unwrap();
+                    let downloaded = download_file(&url, &path, Some(&hash), false);
+                    if downloaded.is_ok() && upload {
+                        let uploaded =
+                            s3cmd.upload_folder(dist_path.to_str().unwrap(), &bucket_name);
+                        if uploaded.is_ok() && delete_after_upload {
+                            fs::remove_file(&path).unwrap();
+                        }
+                    }
                 });
             });
             pool.join();
@@ -204,4 +220,3 @@ pub fn compare_date(entry: &DirEntry, sync_days: i64) -> bool {
         false
     }
 }
-

src/crates/crates_file.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 use walkdir::{DirEntry, WalkDir};
 
 use std::collections::BTreeMap;
-use std::fs::{File, OpenOptions};
+use std::fs::{File, OpenOptions, self};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::{Path, PathBuf};
 use std::str;
@@ -16,7 +16,7 @@ use threadpool::ThreadPool;
 use crate::cloud::s3::{CloudStorage, S3cmd};
 use crate::config::CratesConfig;
 use crate::crates::index;
-use crate::download::{download_file, upload_file};
+use crate::download::download_file;
 use crate::errors::FreightResult;
 
 use super::index::CrateIndex;
@@ -40,6 +40,8 @@ pub struct CratesOptions {
     pub log_path: PathBuf,
 
     pub bucket_name: String,
+
+    pub delete_after_upload: bool,
 }
 
 /// Crate preserve the crates info parse from registry json file
@@ -217,7 +219,7 @@ pub fn parse_index_and_download(
                     .join(format!("{}-{}.crate", &c.name, &c.vers));
 
                 pool.execute(move || {
-                    download_crates_with_log(file, opts.upload, url, c, err_record);
+                    download_crates_with_log(file, &opts, url, c, err_record);
                 });
             }
         }
@@ -239,20 +241,21 @@ pub fn parse_index_and_download(
 
 pub fn download_crates_with_log(
     file: PathBuf,
-    upload: bool,
+    opts: &CratesOptions,
     url: String,
     c: Crate,
     err_record: Arc<Mutex<File>>,
 ) {
     match download_file(&url, &file, Some(&c.cksum), false) {
         Ok(download_succ) => {
-            if download_succ && upload {
-                upload_file(
-                    file.to_str().unwrap(),
-                    &c.name,
-                    format!("{}-{}.crate", &c.name, &c.vers).as_str(),
-                )
-                .unwrap();
+            if download_succ && opts.upload {
+                let s3 = S3cmd::default();
+                let s3_path = file.to_str().unwrap().replace(opts.crates_path.to_str().unwrap(), "");
+                info!("s3_path: {}, {}", s3_path, opts.delete_after_upload);
+                let uploded = s3.upload_file(&file, &s3_path, &opts.bucket_name);
+                if uploded.is_ok() && opts.delete_after_upload {
+                    fs::remove_file(file).unwrap();
+                }
             }
         }
         Err(err) => {
@@ -268,6 +271,4 @@ pub fn download_crates_with_log(
             error!("{:?}", err);
         }
     }
-
-
 }