Add archive deletion job

config.defaults.yml

@@ -12,6 +12,7 @@ defaultDiskUsageLimit: 100mb
 jobs:
   popularArchivesIndex: 30s
   userDiskUsage: 5m
+  deleteDeadArchives: 5m
 
 # user settings
 registration:

lib/archiver.js

@@ -5,6 +5,7 @@ var datEncoding = require('dat-encoding')
 var discoverySwarm = require('discovery-swarm')
 var swarmDefaults = require('datland-swarm-defaults')
 var mkdirp = require('mkdirp')
+var rimraf = require('rimraf')
 var getFolderSize = require('get-folder-size')
 var ms = require('ms')
 var pda = require('pauls-dat-api')
@@ -13,6 +14,7 @@ var debug = require('debug')('archiver')
 var debugJobs = require('debug')('jobs')
 
 mkdirp = promisify(mkdirp)
+rimraf = promisify(rimraf)
 getFolderSize = promisify(getFolderSize)
 
 // exported api
@@ -29,6 +31,7 @@ module.exports = class Archiver {
     this.indexes = {popular: []}
     this._startJob(this.computePopularIndex, 'popularArchivesIndex')
     this._startJob(this.computeUserDiskUsageAndSwarm, 'userDiskUsage')
+    this._startJob(this.deleteDeadArchives, 'deleteDeadArchives')
   }
 
   // methods
@@ -175,6 +178,26 @@ module.exports = class Archiver {
     }
   }
 
+  async deleteDeadArchives () {
+    var release = await lock('archiver-job')
+    try {
+      debugJobs('START Delete dead archives')
+      var start = Date.now()
+      var deadArchiveKeys = await this.cloud.archivesDB.listDeadArchiveKeys()
+      await Promise.all(deadArchiveKeys.map(async (archiveKey) => {
+        // delete files
+        var archivePath = this._getArchiveFilesPath(archiveKey)
+        await rimraf(archivePath, {disableGlob: true})
+      }))
+    } catch (e) {
+      console.error(e)
+      debugJobs('FAILED Delete dead archives (%dms)', (Date.now() - start))
+    } finally {
+      debugJobs('FINISH Delete dead archives (%dms)', (Date.now() - start))
+      release()
+    }
+  }
+
   // internal
   // =
 

lib/dbs/archives.js

@@ -210,8 +210,13 @@ class ArchivesDB extends EventEmitter {
   // internal tracking
   // =
 
-  listDeadArchives () {
-    return this.deadArchivesDB.createKeyStream()
+  listDeadArchiveKeys () {
+    return new Promise((resolve, reject) => {
+      collect(this.deadArchivesDB.createKeyStream(), (err, res) => {
+        if (err) reject(err)
+        else resolve(res)
+      })
+    })
   }
 
   async updateDeadArchives (key, numHostingUsers) {

package.json

@@ -64,6 +64,7 @@
     "pretty-bytes": "^4.0.2",
     "request": "^2.79.0",
     "request-promise-native": "^1.0.3",
+    "rimraf": "^2.6.1",
     "stream-collector": "^1.0.1",
     "subleveldown": "^2.1.0",
     "through2": "^2.0.3",

test/archives.js

@@ -1,11 +1,14 @@
 var test = require('ava')
 var path = require('path')
+var fs = require('fs')
+var promisify = require('es6-promisify')
 var createTestServer = require('./lib/server.js')
 var { makeDatFromFolder, downloadDatFromSwarm } = require('./lib/dat.js')
 
 var app
 var sessionToken, auth, authUser
 var testDat, testDatKey
+var fsstat = promisify(fs.stat, fs)
 
 test.cb('start test server', t => {
   app = createTestServer(async err => {
@@ -370,6 +373,18 @@ test('check archive status after removed', async t => {
   t.is(res.statusCode, 404, '404 not found')
 })
 
+test('delete dead archives job', async t => {
+  // folder exists
+  var stat = await fsstat(app.cloud.archiver._getArchiveFilesPath(testDatKey))
+  t.truthy(stat)
+
+  // run job
+  await app.cloud.archiver.deleteDeadArchives()
+
+  // folder does not exist
+  await t.throws(fsstat(app.cloud.archiver._getArchiveFilesPath(testDatKey)))
+})
+
 test('archive status wont stall on archive that fails to sync', async t => {
   // add a fake archive
   var fakeKey = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'

test/lib/server.js

@@ -40,7 +40,8 @@ function createLocalApp (cb) {
     },
     jobs: {
       popularArchivesIndex: '15m',
-      userDiskUsage: '30m'
+      userDiskUsage: '30m',
+      deleteDeadArchives: '30m'
     },
     registration: {
       open: true,