Refactor static file handling

Cargo.toml

@@ -66,7 +66,6 @@ serde_json = "1.0"
 # iron dependencies
 iron = "0.5"
 router = "0.5"
-staticfile = { version = "0.4", features = ["cache"] }
 tempfile = "3.1.0"
 
 # Templating

build.rs

@@ -1,11 +1,5 @@
 use git2::Repository;
-use std::{
-    env,
-    error::Error,
-    fs::{self, File},
-    io::Write,
-    path::Path,
-};
+use std::{env, error::Error, fs::File, io::Write, path::Path};
 
 fn main() {
     // Don't rerun anytime a single change is made
@@ -15,8 +9,6 @@ fn main() {
     println!("cargo:rerun-if-changed=templates/style/_vars.scss");
     println!("cargo:rerun-if-changed=templates/style/_utils.scss");
     println!("cargo:rerun-if-changed=templates/style/_navbar.scss");
-    println!("cargo:rerun-if-changed=templates/menu.js");
-    println!("cargo:rerun-if-changed=templates/index.js");
     println!("cargo:rerun-if-changed=vendor/");
     // TODO: are these right?
     println!("cargo:rerun-if-changed=.git/HEAD");
@@ -26,7 +18,6 @@ fn main() {
     if let Err(sass_err) = compile_sass() {
         panic!("Error compiling sass: {}", sass_err);
     }
-    copy_js();
 }
 
 fn write_git_version() {
@@ -95,12 +86,3 @@ fn compile_sass() -> Result<(), Box<dyn Error>> {
 
     Ok(())
 }
-
-fn copy_js() {
-    ["menu.js", "index.js"].iter().for_each(|path| {
-        let source_path =
-            Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).join(format!("templates/{}", path));
-        let dest_path = Path::new(&env::var("OUT_DIR").unwrap()).join(path);
-        fs::copy(&source_path, &dest_path).expect("Copy JavaScript file to target");
-    });
-}

dockerfiles/Dockerfile

@@ -51,8 +51,6 @@ RUN touch build.rs
 COPY src src/
 RUN find src -name "*.rs" -exec touch {} \;
 COPY templates/style templates/style
-COPY templates/index.js templates/
-COPY templates/menu.js templates/
 COPY vendor vendor/
 
 RUN cargo build --release
@@ -76,7 +74,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 RUN mkdir -p /opt/docsrs/prefix
 
 COPY --from=build /build/target/release/cratesfyi /usr/local/bin
-COPY static /opt/docsrs/prefix/public_html
+COPY static /opt/docsrs/static
 COPY templates /opt/docsrs/templates
 COPY dockerfiles/entrypoint.sh /opt/docsrs/
 COPY vendor /opt/docsrs/vendor

src/web/error.rs

@@ -144,7 +144,7 @@ mod tests {
                     .next()
                     .unwrap()
                     .text_contents(),
-                "The requested resource does not exist",
+                "The requested crate does not exist",
             );
 
             Ok(())
@@ -170,7 +170,7 @@ mod tests {
                     .next()
                     .unwrap()
                     .text_contents(),
-                "The requested resource does not exist",
+                "The requested crate does not exist",
             );
 
             Ok(())
@@ -190,7 +190,7 @@ mod tests {
                     .next()
                     .unwrap()
                     .text_contents(),
-                "The requested resource does not exist",
+                "The requested crate does not exist",
             );
 
             Ok(())
@@ -209,7 +209,7 @@ mod tests {
                     .next()
                     .unwrap()
                     .text_contents(),
-                "The requested resource does not exist",
+                "The requested crate does not exist",
             );
 
             Ok(())
@@ -232,7 +232,7 @@ mod tests {
                     .next()
                     .unwrap()
                     .text_contents(),
-                "The requested resource does not exist",
+                "The requested crate does not exist",
             );
 
             Ok(())

src/web/file.rs

@@ -1,7 +1,7 @@
 //! Database based file handler
 
 use crate::storage::{Blob, Storage};
-use crate::{error::Result, Config};
+use crate::{error::Result, Config, Metrics};
 use iron::{status, Handler, IronResult, Request, Response};
 
 #[derive(Debug)]
@@ -60,6 +60,15 @@ impl Handler for DatabaseFileHandler {
         let storage = extension!(req, Storage);
         let config = extension!(req, Config);
         if let Ok(file) = File::from_path(&storage, &path, &config) {
+            let metrics = extension!(req, Metrics);
+
+            // Because all requests that don't hit another handler go through here, we will get all
+            // requests successful or not recorded by the RequestRecorder, so we inject an extra
+            // "database success" route to keep track of how often we succeed vs 404
+            metrics
+                .routes_visited
+                .with_label_values(&["database success"])
+                .inc();
             Ok(file.serve())
         } else {
             Err(super::error::Nope::CrateNotFound.into())

src/web/metrics.rs

@@ -119,7 +119,7 @@ mod tests {
             ("/crate/rcc/0.0.0", "/crate/:name/:version"),
             ("/-/static/index.js", "static resource"),
             ("/-/static/menu.js", "static resource"),
-            ("/opensearch.xml", "static resource"),
+            ("/-/static/opensearch.xml", "static resource"),
             ("/releases", "/releases"),
             ("/releases/feed", "static resource"),
             ("/releases/queue", "/releases/queue"),
@@ -129,10 +129,12 @@ mod tests {
                 "/releases/recent-failures/:page",
             ),
             ("/releases/recent/1", "/releases/recent/:page"),
-            ("/robots.txt", "static resource"),
+            ("/-/static/robots.txt", "static resource"),
             ("/sitemap.xml", "static resource"),
             ("/-/static/style.css", "static resource"),
             ("/-/static/vendored.css", "static resource"),
+            ("/rustdoc/rcc/0.0.0/rcc/index.html", "database"),
+            ("/rustdoc/gcc/0.0.0/gcc/index.html", "database"),
         ];
 
         wrapper(|env| {
@@ -179,6 +181,15 @@ mod tests {
                 );
             }
 
+            // extra metrics for the "database success" hack
+            assert_eq!(
+                metrics
+                    .routes_visited
+                    .with_label_values(&["database success"])
+                    .get(),
+                2
+            );
+
             Ok(())
         })
     }

src/web/mod.rs

@@ -96,31 +96,29 @@ use extensions::InjectExtensions;
 use failure::Error;
 use iron::{
     self,
-    headers::{CacheControl, CacheDirective, ContentType, Expires, HttpDate},
+    headers::{Expires, HttpDate},
     modifiers::Redirect,
     status,
     status::Status,
     Chain, Handler, Iron, IronError, IronResult, Listening, Request, Response, Url,
 };
+use metrics::RequestRecorder;
 use page::TemplateData;
 use postgres::Client;
 use router::NoRoute;
 use semver::{Version, VersionReq};
 use serde::Serialize;
-use staticfile::Static;
-use std::{borrow::Cow, env, fmt, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
+use std::{borrow::Cow, fmt, net::SocketAddr, sync::Arc};
 
 /// Duration of static files for staticfile and DatabaseFileHandler (in seconds)
 const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 30 * 12; // 12 months
-const OPENSEARCH_XML: &[u8] = include_bytes!("opensearch.xml");
 
 const DEFAULT_BIND: &str = "0.0.0.0:3000";
 
 struct CratesfyiHandler {
     shared_resource_handler: Box<dyn Handler>,
     router_handler: Box<dyn Handler>,
     database_file_handler: Box<dyn Handler>,
-    static_handler: Box<dyn Handler>,
     inject_extensions: InjectExtensions,
 }
 
@@ -144,22 +142,14 @@ impl CratesfyiHandler {
         let shared_resources =
             Self::chain(inject_extensions.clone(), rustdoc::SharedResourceHandler);
         let router_chain = Self::chain(inject_extensions.clone(), routes.iron_router());
-        let prefix = PathBuf::from(
-            env::var("CRATESFYI_PREFIX")
-                .expect("the CRATESFYI_PREFIX environment variable is not set"),
-        )
-        .join("public_html");
-        let static_handler =
-            Static::new(prefix).cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION));
 
         Ok(CratesfyiHandler {
             shared_resource_handler: Box::new(shared_resources),
             router_handler: Box::new(router_chain),
             database_file_handler: Box::new(routes::BlockBlacklistedPrefixes::new(
                 blacklisted_prefixes,
-                Box::new(file::DatabaseFileHandler),
+                Box::new(RequestRecorder::new(file::DatabaseFileHandler, "database")),
             )),
-            static_handler: Box::new(static_handler),
             inject_extensions,
         })
     }
@@ -185,7 +175,6 @@ impl Handler for CratesfyiHandler {
             .handle(req)
             .or_else(|e| if_404(e, || self.router_handler.handle(req)))
             .or_else(|e| if_404(e, || self.database_file_handler.handle(req)))
-            .or_else(|e| if_404(e, || self.static_handler.handle(req)))
             .or_else(|e| {
                 let err = if let Some(err) = e.error.downcast::<error::Nope>() {
                     *err
@@ -518,21 +507,6 @@ fn redirect_base(req: &Request) -> String {
     }
 }
 
-fn opensearch_xml_handler(_: &mut Request) -> IronResult<Response> {
-    let mut response = Response::with((status::Ok, OPENSEARCH_XML));
-    let cache = vec![
-        CacheDirective::Public,
-        CacheDirective::MaxAge(STATIC_FILE_CACHE_DURATION as u32),
-    ];
-    response.headers.set(ContentType(
-        "application/opensearchdescription+xml".parse().unwrap(),
-    ));
-
-    response.headers.set(CacheControl(cache));
-
-    Ok(response)
-}
-
 /// MetaData used in header
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub(crate) struct MetaData {