many: implement fetching sections and package names periodically. (#3866

)

These are then used for quicker tab completion.

cmd/snap/cmd_find.go

@@ -27,6 +27,7 @@ import (
 	"github.com/jessevdk/go-flags"
 
 	"github.com/snapcore/snapd/client"
+	"github.com/snapcore/snapd/dirs"
 	"github.com/snapcore/snapd/i18n"
 )
 
@@ -68,6 +69,10 @@ func getPrice(prices map[string]float64, currency string) (float64, string, erro
 type SectionName string
 
 func (s SectionName) Complete(match string) []flags.Completion {
+	if ret, err := completeFromSortedFile(dirs.SnapSectionsFile, match); err == nil {
+		return ret
+	}
+
 	cli := Client()
 	sections, err := cli.Sections()
 	if err != nil {

cmd/snap/complete.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/jessevdk/go-flags"
 
 	"github.com/snapcore/snapd/asserts"
 	"github.com/snapcore/snapd/client"
+	"github.com/snapcore/snapd/dirs"
 )
 
 type installedSnapName string
@@ -28,9 +31,46 @@ func (s installedSnapName) Complete(match string) []flags.Completion {
 	return ret
 }
 
+func completeFromSortedFile(filename, match string) ([]flags.Completion, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var ret []flags.Completion
+
+	// TODO: look into implementing binary search
+	//       e.g. https://github.com/pts/pts-line-bisect/
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line < match {
+			continue
+		}
+		if !strings.HasPrefix(line, match) {
+			break
+		}
+		ret = append(ret, flags.Completion{Item: line})
+		if len(ret) > 10000 {
+			// too many matches; slow machines could take too long to process this
+			// e.g. the bbb takes ~1s to process ~2M entries (i.e. to reach the
+			// point of asking the user if they actually want to see that many
+			// results). 10k ought to be enough for anybody.
+			break
+		}
+	}
+
+	return ret, nil
+}
+
 type remoteSnapName string
 
 func (s remoteSnapName) Complete(match string) []flags.Completion {
+	if ret, err := completeFromSortedFile(dirs.SnapNamesFile, match); err == nil {
+		return ret
+	}
+
 	if len(match) < 3 {
 		return nil
 	}

daemon/api_test.go

@@ -158,8 +158,6 @@ func (s *apiBaseSuite) muxVars(*http.Request) map[string]string {
 func (s *apiBaseSuite) SetUpSuite(c *check.C) {
 	muxVars = s.muxVars
 	s.restoreRelease = release.MockForcedDevmode(false)
-
-	snapstate.CanAutoRefresh = nil
 	s.systemctlRestorer = systemd.MockSystemctl(s.systemctl)
 	s.journalctlRestorer = systemd.MockJournalctl(s.journalctl)
 }
@@ -290,6 +288,11 @@ func (s *apiBaseSuite) daemon(c *check.C) *Daemon {
 		Serial: "serialserial",
 	})
 
+	// don't actually try to talk to the store on snapstate.Ensure
+	// needs doing after the call to devicestate.Manager (which
+	// happens in daemon.New via overlord.New)
+	snapstate.CanAutoRefresh = nil
+
 	s.d = d
 	return d
 }

daemon/daemon_test.go

@@ -41,7 +41,6 @@ import (
 	"github.com/snapcore/snapd/dirs"
 	"github.com/snapcore/snapd/logger"
 	"github.com/snapcore/snapd/overlord/auth"
-	"github.com/snapcore/snapd/overlord/snapstate"
 	"github.com/snapcore/snapd/overlord/state"
 	"github.com/snapcore/snapd/polkit"
 	"github.com/snapcore/snapd/testutil"
@@ -63,10 +62,6 @@ func (s *daemonSuite) checkAuthorizationForPid(pid uint32, actionId string, deta
 	return s.authorized, s.err
 }
 
-func (s *daemonSuite) SetUpSuite(c *check.C) {
-	snapstate.CanAutoRefresh = nil
-}
-
 func (s *daemonSuite) SetUpTest(c *check.C) {
 	dirs.SetRootDir(c.MkDir())
 	err := os.MkdirAll(filepath.Dir(dirs.SnapStateFile), 0755)

dirs/dirs.go

@@ -69,6 +69,10 @@ var (
 	SnapRepairRunDir     string
 	SnapRepairAssertsDir string
 
+	SnapCacheDir     string
+	SnapNamesFile    string
+	SnapSectionsFile string
+
 	SnapBinariesDir     string
 	SnapServicesDir     string
 	SnapDesktopFilesDir string
@@ -187,6 +191,10 @@ func SetRootDir(rootdir string) {
 
 	SnapStateFile = filepath.Join(rootdir, snappyDir, "state.json")
 
+	SnapCacheDir = filepath.Join(rootdir, "/var/cache/snapd")
+	SnapNamesFile = filepath.Join(SnapCacheDir, "names")
+	SnapSectionsFile = filepath.Join(SnapCacheDir, "sections")
+
 	SnapSeedDir = filepath.Join(rootdir, snappyDir, "seed")
 	SnapDeviceDir = filepath.Join(rootdir, snappyDir, "device")
 

overlord/devicestate/firstboot_test.go

@@ -107,6 +107,10 @@ func (s *FirstBootTestSuite) SetUpTest(c *C) {
 	ovld, err := overlord.New()
 	c.Assert(err, IsNil)
 	s.overlord = ovld
+
+	// don't actually try to talk to the store on snapstate.Ensure
+	// needs doing after the call to devicestate.Manager (which happens in overlord.New)
+	snapstate.CanAutoRefresh = nil
 }
 
 func (s *FirstBootTestSuite) TearDownTest(c *C) {
@@ -451,6 +455,7 @@ func (s *FirstBootTestSuite) TestPopulateFromSeedMissingBootloader(c *C) {
 	snapmgr, err := snapstate.Manager(st)
 	c.Assert(err, IsNil)
 	o.AddManager(snapmgr)
+
 	ifacemgr, err := ifacestate.Manager(st, nil, nil, nil)
 	c.Assert(err, IsNil)
 	o.AddManager(ifacemgr)

overlord/managers_test.go

@@ -116,7 +116,6 @@ func (ms *mgrsSuite) SetUpTest(c *C) {
 	}
 
 	os.Setenv("SNAPPY_SQUASHFS_UNPACK_FOR_TESTS", "1")
-	snapstate.CanAutoRefresh = nil
 
 	// create a fake systemd environment
 	os.MkdirAll(filepath.Join(dirs.SnapServicesDir, "multi-user.target.wants"), 0755)
@@ -191,6 +190,9 @@ func (ms *mgrsSuite) SetUpTest(c *C) {
 		Current:  snap.R(1),
 		SnapType: "os",
 	})
+	// don't actually try to talk to the store on snapstate.Ensure
+	// needs doing after the call to devicestate.Manager (which happens in overlord.New)
+	snapstate.CanAutoRefresh = nil
 }
 
 func (ms *mgrsSuite) TearDownTest(c *C) {

overlord/snapstate/backend_test.go

@@ -22,6 +22,7 @@ package snapstate_test
 import (
 	"errors"
 	"fmt"
+	"io"
 	"sort"
 	"strings"
 
@@ -269,6 +270,24 @@ func (f *fakeStore) Download(ctx context.Context, name, targetFn string, snapInf
 	return nil
 }
 
+func (f *fakeStore) WriteCatalogs(io.Writer) error {
+	f.pokeStateLock()
+	f.fakeBackend.ops = append(f.fakeBackend.ops, fakeOp{
+		op: "x-commands",
+	})
+
+	return nil
+}
+
+func (f *fakeStore) Sections(*auth.UserState) ([]string, error) {
+	f.pokeStateLock()
+	f.fakeBackend.ops = append(f.fakeBackend.ops, fakeOp{
+		op: "x-sections",
+	})
+
+	return nil, nil
+}
+
 type fakeSnappyBackend struct {
 	ops fakeOps
 

overlord/snapstate/snapmgr.go

@@ -35,6 +35,7 @@ import (
 	"github.com/snapcore/snapd/overlord/configstate/config"
 	"github.com/snapcore/snapd/overlord/snapstate/backend"
 	"github.com/snapcore/snapd/overlord/state"
+	"github.com/snapcore/snapd/overlord/storestate"
 	"github.com/snapcore/snapd/release"
 	"github.com/snapcore/snapd/snap"
 	"github.com/snapcore/snapd/strutil"
@@ -68,6 +69,7 @@ const defaultRefreshSchedule = "00:00-04:59/5:00-10:59/11:00-16:59/17:00-23:59"
 
 // overridden in the tests
 var errtrackerReport = errtracker.Report
+var catalogRefreshDelay = 24 * time.Hour
 
 // SnapManager is responsible for the installation and removal of snaps.
 type SnapManager struct {
@@ -78,6 +80,8 @@ type SnapManager struct {
 	nextRefresh            time.Time
 	lastRefreshAttempt     time.Time
 
+	nextCatalogRefresh time.Time
+
 	lastUbuntuCoreTransitionAttempt time.Time
 
 	runner *state.TaskRunner
@@ -448,14 +452,26 @@ func (m *SnapManager) LastRefresh() (time.Time, error) {
 	return lastRefresh, nil
 }
 
+// NextRefresh returns the time the next update of the system's snaps
+// will be attempted.
+// The caller should be holding the state lock.
 func (m *SnapManager) NextRefresh() time.Time {
 	return m.nextRefresh
 }
 
+// RefreshSchedule returns the current refresh schedule.
+// The caller should be holding the state lock.
 func (m *SnapManager) RefreshSchedule() string {
 	return m.currentRefreshSchedule
 }
 
+// NextCatalogRefresh returns the time the next update of catalog
+// data will be attempted.
+// The caller should be holding the state lock.
+func (m *SnapManager) NextCatalogRefresh() time.Time {
+	return m.nextCatalogRefresh
+}
+
 // ensureRefreshes ensures that we refresh all installed snaps periodically
 func (m *SnapManager) ensureRefreshes() error {
 	m.state.Lock()
@@ -518,6 +534,33 @@ func (m *SnapManager) ensureRefreshes() error {
 	return err
 }
 
+// ensureCatalogRefresh ensures that we refresh the catalog
+// data periodically
+func (m *SnapManager) ensureCatalogRefresh() error {
+	// sneakily don't do anything if in testing
+	if CanAutoRefresh == nil {
+		return nil
+	}
+	m.state.Lock()
+	defer m.state.Unlock()
+
+	theStore := storestate.Store(m.state)
+	now := time.Now()
+	needsRefresh := m.nextCatalogRefresh.IsZero() || m.nextCatalogRefresh.Before(now)
+
+	if !needsRefresh {
+		return nil
+	}
+
+	next := now.Add(catalogRefreshDelay)
+	// catalog refresh does not carry on trying on error
+	m.nextCatalogRefresh = next
+
+	logger.Debugf("Catalog refresh starting now; next scheduled for %s.", next)
+
+	return refreshCatalogs(m.state, theStore)
+}
+
 // ensureForceDevmodeDropsDevmodeFromState undoes the froced devmode
 // in snapstate for forced devmode distros.
 func (m *SnapManager) ensureForceDevmodeDropsDevmodeFromState() error {
@@ -667,6 +710,7 @@ func (m *SnapManager) Ensure() error {
 		m.ensureForceDevmodeDropsDevmodeFromState(),
 		m.ensureUbuntuCoreTransition(),
 		m.ensureRefreshes(),
+		m.ensureCatalogRefresh(),
 	}
 
 	m.runner.Ensure()

overlord/snapstate/snapstate.go

@@ -23,14 +23,17 @@ package snapstate
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"reflect"
 	"sort"
+	"strings"
 
 	"github.com/snapcore/snapd/boot"
 	"github.com/snapcore/snapd/dirs"
 	"github.com/snapcore/snapd/i18n"
 	"github.com/snapcore/snapd/interfaces"
 	"github.com/snapcore/snapd/logger"
+	"github.com/snapcore/snapd/osutil"
 	"github.com/snapcore/snapd/overlord/auth"
 	"github.com/snapcore/snapd/overlord/state"
 	"github.com/snapcore/snapd/overlord/storestate"
@@ -1696,3 +1699,33 @@ func ConfigDefaults(st *state.State, snapName string) (map[string]interface{}, e
 
 	return defaults, nil
 }
+
+func refreshCatalogs(st *state.State, theStore storestate.StoreService) error {
+	st.Unlock()
+	defer st.Lock()
+
+	if err := os.MkdirAll(dirs.SnapCacheDir, 0755); err != nil {
+		return fmt.Errorf("cannot create directory %q: %v", dirs.SnapCacheDir, err)
+	}
+
+	sections, err := theStore.Sections(nil)
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(sections)
+	if err := osutil.AtomicWriteFile(dirs.SnapSectionsFile, []byte(strings.Join(sections, "\n")), 0644, 0); err != nil {
+		return err
+	}
+
+	namesFile, err := osutil.NewAtomicFile(dirs.SnapNamesFile, 0644, 0, -1, -1)
+	if err != nil {
+		return err
+	}
+	defer namesFile.Cancel()
+	if err := theStore.WriteCatalogs(namesFile); err != nil {
+		return err
+	}
+
+	return namesFile.Commit()
+}

overlord/storestate/storestate.go

@@ -21,6 +21,7 @@ package storestate
 
 import (
 	"fmt"
+	"io"
 	"net/url"
 
 	"golang.org/x/net/context"
@@ -68,6 +69,7 @@ type StoreService interface {
 	LookupRefresh(*store.RefreshCandidate, *auth.UserState) (*snap.Info, error)
 	ListRefresh([]*store.RefreshCandidate, *auth.UserState) ([]*snap.Info, error)
 	Sections(user *auth.UserState) ([]string, error)
+	WriteCatalogs(names io.Writer) error
 	Download(context.Context, string, string, *snap.DownloadInfo, progress.Meter, *auth.UserState) error
 
 	Assertion(assertType *asserts.AssertionType, primaryKey []string, user *auth.UserState) (asserts.Assertion, error)

packaging/fedora/snap-mgmt.sh

@@ -105,6 +105,9 @@ purge() {
     rm -rf /var/lib/snapd/seccomp/bpf/*
     rm -rf /var/lib/snapd/device/*
     rm -rf /var/lib/snapd/assertions/*
+
+    echo "Removing snapd catalog cache"
+    rm -f /var/cache/snapd/*
 }
 
 while [ -n "$1" ]; do