Recent download stats by minor version

app/components/versions-graph.js

@@ -0,0 +1,125 @@
+import major from 'semver/functions/major';
+import minor from 'semver/functions/minor';
+import lt from 'semver/functions/lt';
+import fetch from 'fetch';
+import Component from '@ember/component';
+
+// Component to plot downloadeds split by semver minor version.
+export default Component.extend({
+  resizeHandler: undefined,
+
+  didInsertElement() {
+    this._super(...arguments);
+
+    this.resizeHandler = () => this.rerender();
+    window.addEventListener('resize', this.resizeHandler, false);
+    document.addEventListener('googleChartsLoaded', this.resizeHandler, false);
+  },
+
+  willDestroyElement() {
+    window.removeEventListener('resize', this.resizeHandler);
+    document.removeEventListener('googleChartsLoaded', this.resizeHandler);
+  },
+
+  didRender() {
+    this._super(...arguments);
+
+    let data = this.model.versions;
+
+    // Early exit if the google plotting libraries do not exist
+    let show = data && window.google && window.googleChartsLoaded;
+    this.element.style.display = show ? '' : 'none';
+    if (!show) {
+      return;
+    }
+
+    // Fetch the recent downloads from the API
+    fetch(`/api/v1/crates/${this.model.name}/recent_downloads`)
+      .then(async r => {
+        if (!r.ok) {
+          console.error('error fetching recent downloads from API');
+          return;
+        }
+
+        let recentDownloads;
+        try {
+          recentDownloads = await r.json();
+        } catch (e) {
+          console.error(e);
+          return;
+        }
+
+        // Build up the list of unique major.minor versions of this crate, and total
+        // up the number of downloads for each semver version
+        let downloadsPerVersion = new Map();
+        recentDownloads.downloads.forEach(v => {
+          let mj = major(v.version);
+          let mn = minor(v.version);
+          let downloads = v.downloads;
+
+          // XXX ugly hack to get semver to parse the version correctly later on.
+          // We want to do a semver-aware sort, but the `semver.lt` function only
+          // understands version triples, not doubles.
+          let key = `${mj}.${mn}.0`;
+          if (downloadsPerVersion.has(key)) {
+            let old = downloadsPerVersion.get(key);
+            downloadsPerVersion.set(key, old + downloads);
+          } else {
+            downloadsPerVersion.set(key, downloads);
+          }
+        });
+
+        // Build up the plotting data
+        let plotData = [
+          // Headings and the nature of additional parameters
+          ['Version', 'Downloads', { role: 'style' }, { role: 'annotation' }],
+        ];
+
+        // Update plotData with rows in the correct format for google visualization library
+        for (let [key, value] of sortIncreasingBySemver(downloadsPerVersion)) {
+          plotData.push([key, value, '#62865f', value]);
+        }
+
+        let myData = window.google.visualization.arrayToDataTable(plotData);
+
+        // Plot options
+        let options = {
+          chart: {
+            title: 'Downloads',
+          },
+          chartArea: { left: 85, width: '77%', height: '80%' },
+          hAxis: {
+            minorGridlines: { count: 8 },
+          },
+          vAxis: {
+            minorGridlines: { count: 5 },
+            viewWindow: { min: 0 },
+          },
+          legend: { position: 'none' },
+        };
+
+        // Draw the plot into the current element
+        let chart = new window.google.visualization.BarChart(this.element);
+        chart.draw(myData, options);
+      })
+      .catch(e => console.error(`Error fetching data from API: ${e}`));
+  },
+});
+
+function sortIncreasingBySemver(downloadsMap) {
+  const items = Array.from(downloadsMap.entries());
+  // Sort by semver comparison
+  items.sort(([versionA], [versionB]) => {
+    // Index 0 is the version string in the array.
+    //
+    // We use `lt` here as we want the array to be sorted in reverse order
+    // (newest at the top)
+    return lt(versionA, versionB);
+  });
+
+  // Update the labels to show e.g. `0.1.x` instead of `0.1.0` which is
+  // required by semver comparisons
+  return items.map(([version, count]) => {
+    return [version.replace(/\.0$/, '.x'), count];
+  });
+}

app/templates/crate/versions.hbs

@@ -10,7 +10,12 @@
   </span>
 </div>
 
-<div local-class='list' class='white-rows'>
+<div class="graph">
+  <h4>Downloads per minor version in the last 90 days</h4>
+  {{versions-graph model=this.model}}
+</div>
+
+<div local-class='list' id='crate-all-versions' class='white-rows'>
   {{#each this.model.versions as |version|}}
     <div class='row'>
       <div>

mirage/route-handlers/crates.js

@@ -58,6 +58,15 @@ export function register(server) {
     return crate.versions.sort((a, b) => compareIsoDates(b.created_at, a.created_at));
   });
 
+  server.get('/api/v1/crates/:crate_id/recent_downloads', (schema, request) => {
+    let crate = request.params.crate_id;
+    let downloads = [
+      { version: '0.1.0', downloads: 32 },
+      { version: '0.2.0', downloads: 128 },
+    ];
+    return { downloads: downloads, meta: { crate: crate, ndays: 90 } };
+  });
+
   server.get('/api/v1/crates/:crate_id/:version_num/authors', (schema, request) => {
     let crateId = request.params.crate_id;
     let crate = schema.crates.find(crateId);

src/config.rs

@@ -18,6 +18,13 @@ pub struct Config {
     pub api_protocol: String,
     pub publish_rate_limit: PublishRateLimit,
     pub blocked_traffic: Vec<(String, Vec<String>)>,
+    // This variable configures how many days ago is considered for "recent" downloads. This
+    // variable must match what is contained within:
+    //
+    // migrations/2018-04-24-145128_create_recent_crate_downloads/up.sql
+    //
+    // By default it is configured to be set up to be 90 days.
+    pub ndays: i32,
 }
 
 impl Default for Config {
@@ -139,6 +146,7 @@ impl Default for Config {
             api_protocol,
             publish_rate_limit: Default::default(),
             blocked_traffic: blocked_traffic(),
+            ndays: 90,
         }
     }
 }

src/controllers/krate/downloads.rs

@@ -8,26 +8,28 @@ use std::cmp;
 use crate::controllers::frontend_prelude::*;
 
 use crate::models::{Crate, CrateVersions, Version, VersionDownload};
-use crate::schema::version_downloads;
+use crate::schema::{version_downloads, versions};
 use crate::views::EncodableVersionDownload;
 
 use crate::models::krate::to_char;
 
+use diesel::sql_types::BigInt;
+
 /// Handles the `GET /crates/:crate_id/downloads` route.
 pub fn downloads(req: &mut dyn Request) -> AppResult<Response> {
     use diesel::dsl::*;
-    use diesel::sql_types::BigInt;
 
     let crate_name = &req.params()["crate_id"];
     let conn = req.db_read_only()?;
     let krate = Crate::by_name(crate_name).first::<Crate>(&*conn)?;
+    let ndays = req.app().config.ndays;
 
     let mut versions = krate.all_versions().load::<Version>(&*conn)?;
     versions.sort_by(|a, b| b.num.cmp(&a.num));
     let (latest_five, rest) = versions.split_at(cmp::min(5, versions.len()));
 
     let downloads = VersionDownload::belonging_to(latest_five)
-        .filter(version_downloads::date.gt(date(now - 90.days())))
+        .filter(version_downloads::date.gt(date(now - ndays.days())))
         .order(version_downloads::date.asc())
         .load(&*conn)?
         .into_iter()
@@ -40,7 +42,7 @@ pub fn downloads(req: &mut dyn Request) -> AppResult<Response> {
             to_char(version_downloads::date, "YYYY-MM-DD"),
             sum_downloads,
         ))
-        .filter(version_downloads::date.gt(date(now - 90.days())))
+        .filter(version_downloads::date.gt(date(now - ndays.days())))
         .group_by(version_downloads::date)
         .order(version_downloads::date.asc())
         .load::<ExtraDownload>(&*conn)?;
@@ -67,3 +69,53 @@ pub fn downloads(req: &mut dyn Request) -> AppResult<Response> {
         meta,
     }))
 }
+
+/// Handles the `GET /crates/:crate_id/recent_downloads` route.
+pub fn recent_downloads(req: &mut dyn Request) -> AppResult<Response> {
+    use diesel::dsl::*;
+
+    let crate_name = &req.params()["crate_id"];
+    let conn = req.db_conn()?;
+    let krate = Crate::by_name(crate_name).first::<Crate>(&*conn)?;
+    let ndays = req.app().config.ndays;
+
+    // Get the versions for this crate
+    let available_versions = krate.all_versions().load::<Version>(&*conn)?;
+
+    #[derive(Debug, Serialize, Queryable)]
+    struct Download {
+        version: semver::Version,
+        downloads: i64,
+    }
+
+    #[derive(Debug, Serialize)]
+    struct Response<'a> {
+        downloads: Vec<Download>,
+        meta: Meta<'a>,
+    }
+
+    #[derive(Debug, Serialize)]
+    struct Meta<'a> {
+        #[serde(rename = "crate")]
+        krate: &'a str,
+        ndays: i32,
+    }
+
+    // Now get the grouped versions for the last `ndays` days.
+    let sum_downloads = sql::<BigInt>("SUM(version_downloads.downloads)");
+    let downloads = VersionDownload::belonging_to(available_versions.as_slice())
+        .inner_join(versions::table)
+        .select((versions::num, sum_downloads))
+        .filter(version_downloads::date.gt(date(now - ndays.days())))
+        .group_by(versions::num)
+        .order_by(versions::num.asc())
+        .load::<Download>(&*conn)?;
+
+    Ok(req.json(&Response {
+        downloads,
+        meta: Meta {
+            krate: crate_name,
+            ndays,
+        },
+    }))
+}

src/router.rs

@@ -56,6 +56,10 @@ pub fn build_router(app: &App) -> R404 {
         "/crates/:crate_id/downloads",
         C(krate::downloads::downloads),
     );
+    api_router.get(
+        "/crates/:crate_id/recent_downloads",
+        C(krate::downloads::recent_downloads),
+    );
     api_router.get("/crates/:crate_id/versions", C(krate::metadata::versions));
     api_router.put("/crates/:crate_id/follow", C(krate::follow::follow));
     api_router.delete("/crates/:crate_id/follow", C(krate::follow::unfollow));

src/schema.rs

@@ -676,7 +676,7 @@ table! {
 table! {
     /// Representation of the `recent_crate_downloads` view.
     ///
-    /// This data represents the downloads in the last 90 days.
+    /// This data represents the downloads in the last `Config::ndays` days.
     /// This view does not contain realtime data.
     /// It is refreshed by the `update-downloads` script.
     recent_crate_downloads (crate_id) {

src/tests/all.rs

@@ -143,6 +143,7 @@ fn simple_config() -> Config {
         api_protocol: String::from("http"),
         publish_rate_limit: Default::default(),
         blocked_traffic: Default::default(),
+        ndays: 90,
     }
 }
 

src/tests/builders.rs

@@ -202,15 +202,16 @@ impl<'a> CrateBuilder<'a> {
         self
     }
 
-    /// Sets the crate's number of downloads that happened more than 90 days ago. The total
-    /// number of downloads for this crate will be this plus the number of recent downloads.
+    /// Sets the crate's number of downloads that happened more than `Config::ndays` days ago. The
+    /// total number of downloads for this crate will be this plus the number of recent downloads.
     pub fn downloads(mut self, downloads: i32) -> Self {
         self.downloads = Some(downloads);
         self
     }
 
-    /// Sets the crate's number of downloads in the last 90 days. The total number of downloads
-    /// for this crate will be this plus the number of downloads set with the `downloads` method.
+    /// Sets the crate's number of downloads in the last `Config::ndays` days. The total number of
+    /// downloads for this crate will be this plus the number of downloads set with the `downloads`
+    /// method.
     pub fn recent_downloads(mut self, recent_downloads: i32) -> Self {
         self.recent_downloads = Some(recent_downloads);
         self