feat: purge cache after expiration time

Author: Luca Bianconi

.gitignore

@@ -28,3 +28,6 @@ venv
 /docsgen/arduino-cli.exe
 /docs/rpc/*.md
 /docs/commands/*.md
+
+# Delve debugger binary file
+__debug_bin

arduino/sketch/sketch.go

@@ -302,5 +302,10 @@ func GenBuildPath(sketchPath *paths.Path) *paths.Path {
 	}
 	md5SumBytes := md5.Sum([]byte(path))
 	md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
-	return paths.TempDir().Join("arduino", "sketch-"+md5Sum)
+
+	return getSketchesCacheDir().Join(md5Sum)
+}
+
+func getSketchesCacheDir() *paths.Path {
+	return paths.TempDir().Join("arduino", "sketches").Canonical()
 }

arduino/sketch/sketch_test.go

@@ -286,10 +286,10 @@ func TestNewSketchFolderSymlink(t *testing.T) {
 }
 
 func TestGenBuildPath(t *testing.T) {
-	want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
+	want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
 	assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want))
 
-	want = paths.TempDir().Join("arduino", "sketch-D41D8CD98F00B204E9800998ECF8427E")
+	want = paths.TempDir().Join("arduino", "sketches", "D41D8CD98F00B204E9800998ECF8427E")
 	assert.True(t, GenBuildPath(nil).EquivalentTo(want))
 }
 

buildcache/build_cache.go

@@ -0,0 +1,69 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package buildcache
+
+import (
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const lastUsedFileName = ".last-used"
+
+// GetOrCreate retrieves or creates the cache directory at the given path
+// If the cache already exists the lifetime of the cache is extended.
+func GetOrCreate(dir *paths.Path) error {
+	if !dir.Exist() {
+		if err := dir.MkdirAll(); err != nil {
+			return err
+		}
+	}
+
+	return dir.Join(lastUsedFileName).WriteFile([]byte{})
+}
+
+// Purge removes all cache directories within baseDir that have expired
+// To know how long ago a directory has been last used
+// it checks into the .last-used file.
+func Purge(baseDir *paths.Path, ttl time.Duration) {
+	files, err := baseDir.ReadDir()
+	if err != nil {
+		return
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			removeIfExpired(file, ttl)
+		}
+	}
+}
+
+func removeIfExpired(dir *paths.Path, ttl time.Duration) {
+	fileInfo, err := dir.Join().Stat()
+	if err != nil {
+		return
+	}
+	lifeExpectancy := ttl - time.Since(fileInfo.ModTime())
+	if lifeExpectancy > 0 {
+		return
+	}
+	logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs())
+	err = dir.RemoveAll()
+	if err != nil {
+		logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err))
+	}
+}

buildcache/build_cache_test.go

@@ -0,0 +1,79 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package buildcache
+
+import (
+	"testing"
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_UpdateLastUsedFileNotExisting(t *testing.T) {
+	testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
+	require.NoError(t, testBuildDir.MkdirAll())
+	timeBeforeUpdating := time.Unix(0, 0)
+	requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
+}
+
+func Test_UpdateLastUsedFileExisting(t *testing.T) {
+	testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
+	require.NoError(t, testBuildDir.MkdirAll())
+
+	// create the file
+	preExistingFile := testBuildDir.Join(lastUsedFileName)
+	require.NoError(t, preExistingFile.WriteFile([]byte{}))
+	timeBeforeUpdating := time.Now().Add(-time.Second)
+	preExistingFile.Chtimes(time.Now(), timeBeforeUpdating)
+	requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
+}
+
+func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) {
+	require.NoError(t, GetOrCreate(dir))
+	expectedFile := dir.Join(lastUsedFileName)
+	fileInfo, err := expectedFile.Stat()
+	require.Nil(t, err)
+	require.Greater(t, fileInfo.ModTime(), prevModTime)
+}
+
+func TestPurge(t *testing.T) {
+	ttl := time.Minute
+
+	dirToPurge := paths.New(t.TempDir(), "root")
+
+	lastUsedTimesByDirPath := map[*paths.Path]time.Time{
+		(dirToPurge.Join("old")):   time.Now().Add(-ttl - time.Hour),
+		(dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute),
+	}
+
+	// create the metadata files
+	for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
+		require.NoError(t, dirPath.MkdirAll())
+		infoFilePath := dirPath.Join(lastUsedFileName).Canonical()
+		require.NoError(t, infoFilePath.WriteFile([]byte{}))
+		// make sure access time does not matter
+		accesstime := time.Now()
+		require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime))
+	}
+
+	Purge(dirToPurge, ttl)
+
+	files, err := dirToPurge.Join("fresh").Stat()
+	require.Nil(t, err)
+	require.True(t, files.IsDir())
+	require.True(t, dirToPurge.Exist())
+}

commands/compile/compile.go

@@ -27,9 +27,11 @@ import (
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/configuration"
 	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/arduino-cli/inventory"
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	if err = builderCtx.BuildPath.MkdirAll(); err != nil {
 		return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err}
 	}
+
+	buildcache.GetOrCreate(builderCtx.BuildPath)
+	// cache is purged after compilation to not remove entries that might be required
+	defer maybePurgeBuildCache()
+
 	builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
 		builderCtx.BuildPath.Join("compile_commands.json"),
 	)
@@ -143,8 +150,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	// Optimize for debug
 	builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
-
-	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache")
+	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
 
 	builderCtx.Jobs = int(req.GetJobs())
 
@@ -284,3 +290,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	return r, nil
 }
+
+// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met.
+func maybePurgeBuildCache() {
+
+	compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge")
+	// 0 means never purge
+	if compilationsBeforePurge == 0 {
+		return
+	}
+	compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge")
+	compilationSinceLastPurge++
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge)
+	defer inventory.WriteStore()
+	if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge {
+		return
+	}
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0)
+	cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs()
+	buildcache.Purge(paths.TempDir().Join("arduino", "cores"), cacheTTL)
+	buildcache.Purge(paths.TempDir().Join("arduino", "sketches"), cacheTTL)
+}

configuration/defaults.go

@@ -18,6 +18,7 @@ package configuration
 import (
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) {
 
 	// Sketch compilation
 	settings.SetDefault("sketch.always_export_binaries", false)
+	settings.SetDefault("build_cache.ttl", time.Hour*24*30)
+	settings.SetDefault("build_cache.compilations_before_purge", 10)
 
 	// daemon settings
 	settings.SetDefault("daemon.port", "50051")

docs/configuration.md

@@ -33,6 +33,12 @@
     to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag.
 - `updater` - configuration options related to Arduino CLI updates
   - `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true`
+- `build_cache` configuration options related to the compilation cache
+  - `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`.
+    When `0` the cache is never purged.
+  - `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files
+    lifetime is renewed. The value format must be a valid input for
+    [time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days).
 
 ## Configuration methods
 

internal/integrationtest/arduino-cli.go

@@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig)
 		"ARDUINO_DATA_DIR":       cli.dataDir.String(),
 		"ARDUINO_DOWNLOADS_DIR":  cli.stagingDir.String(),
 		"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
+		"ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0",
 	}
 	env.RegisterCleanUpCallback(cli.CleanUp)
 	return cli