Skip to content

Commit

Permalink
Using Directories options from #4752 for recursive printing.
Browse files Browse the repository at this point in the history
  • Loading branch information
Samuel Bizien Filippi committed Oct 5, 2018
1 parent fe428e7 commit 18373c7
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 54 deletions.
34 changes: 31 additions & 3 deletions internal/globpath/globpath.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,25 @@ func Compile(path string) (*GlobPath, error) {
hasMeta: hasMeta(path),
hasSuperMeta: hasSuperMeta(path),
path: path,
root: "",
}

// Get the root directory for this filepath
out.root = findRootDir(path)

// if there are no glob meta characters in the path, don't bother compiling
// a glob object or finding the root directory. (see short-circuit in Match)
// a glob object. (see short-circuits in Match and MatchString)
if !out.hasMeta || !out.hasSuperMeta {
if path != "/" {
out.path = strings.TrimSuffix(path, "/")
}
return &out, nil
}

var err error
if out.g, err = glob.Compile(path, os.PathSeparator); err != nil {
return nil, err
}
// Get the root directory for this filepath
out.root = findRootDir(path)
return &out, nil
}

Expand All @@ -64,6 +69,21 @@ func (g *GlobPath) Match() map[string]os.FileInfo {
return walkFilePath(g.root, g.g)
}

func (g *GlobPath) MatchString(path string) bool {
if !g.hasMeta {
return (g.path == path)
}
if !g.hasSuperMeta {
res, _ := filepath.Match(g.path, path)
return res
}
return g.g.Match(path)
}

func (g *GlobPath) GetRootDir() string {
return g.root
}

// walk the filepath from the given root and return a list of files that match
// the given glob.
func walkFilePath(root string, g glob.Glob) map[string]os.FileInfo {
Expand All @@ -81,6 +101,7 @@ func walkFilePath(root string, g glob.Glob) map[string]os.FileInfo {
// find the root dir of the given path (could include globs).
// ie:
// /var/log/telegraf.conf -> /var/log
// /etc/telegraf.d -> /etc/telegraf.d
// /home/** -> /home
// /home/*/** -> /home
// /lib/share/*/*/**.txt -> /lib/share
Expand All @@ -89,6 +110,13 @@ func findRootDir(path string) string {
out := sepStr
for i, item := range pathItems {
if i == len(pathItems)-1 {
file, err := os.Stat(out + item)
if err != nil {
break
}
if file.IsDir() {
out += item
}
break
}
if item == "" {
Expand Down
12 changes: 9 additions & 3 deletions plugins/inputs/filecount/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,17 @@ Counts files in directories that match certain criteria.
# Count files in a directory and compute their size
[[inputs.filecount]]
## Directory to gather stats about.
## deprecated in 1.9; use the directories option
directory = "/var/cache/apt/archives"

## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]

## Also compute total size of matched elements. Defaults to false.
count_size = false

Expand All @@ -34,9 +43,6 @@ Counts files in directories that match certain criteria.
## touched in this duration. Defaults to "0s".
mtime = "0s"

## Output stats for every subdirectory. Defaults to false.
recursive_print = false

## Only output directories whose sub elements weighs more than this
## size. Defaults to "0B".
recursive_print_size = "0B"
Expand Down
67 changes: 45 additions & 22 deletions plugins/inputs/filecount/filecount.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,23 @@ import (
"github.com/alecthomas/units"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/globpath"
"github.com/influxdata/telegraf/plugins/inputs"
)

const sampleConfig = `
## Directory to gather stats about.
## deprecated in 1.9; use the directories option
directory = "/var/cache/apt/archives"
## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]
## Also compute total size of matched elements. Defaults to false.
count_size = false
Expand All @@ -39,24 +49,21 @@ const sampleConfig = `
## touched in this duration. Defaults to "0s".
mtime = "0s"
## Output stats for every subdirectory. Defaults to false.
recursive_print = false
## Only output directories whose sub elements weighs more than this
## size. Defaults to "0B".
recursive_print_size = "0B"
`

type FileCount struct {
Directory string
Directory string // Deprecated in 1.9
Directories []string
CountSize bool
Name string
Recursive bool
RegularOnly bool
Size string
SizeB int64
MTime internal.Duration `toml:"mtime"`
RecursivePrint bool
RecursivePrintSize string
RecursivePrintSizeB int64
fileFilters []fileFilterFunc
Expand Down Expand Up @@ -152,7 +159,7 @@ func (fc *FileCount) initFileFilters() {
fc.fileFilters = rejectNilFilters(filters)
}

func (fc *FileCount) count(acc telegraf.Accumulator, basedir string) (int64, int64) {
func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob *globpath.GlobPath) (int64, int64) {
numFiles, totalSize, nf, ts := int64(0), int64(0), int64(0), int64(0)

directory, err := os.Open(basedir)
Expand All @@ -168,7 +175,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string) (int64, int
}
for _, file := range files {
if fc.Recursive && file.IsDir() {
nf, ts = fc.count(acc, basedir+string(os.PathSeparator)+file.Name())
nf, ts = fc.count(acc, basedir+string(os.PathSeparator)+file.Name(), glob)
numFiles += nf
totalSize += ts
}
Expand All @@ -182,23 +189,20 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string) (int64, int
}
}

if fc.RecursivePrint || basedir == fc.Directory {
if totalSize >= fc.RecursivePrintSizeB || basedir == fc.Directory {
gauge := map[string]interface{}{
"count": numFiles,
}
if fc.CountSize {
gauge["size"] = totalSize
}
acc.AddGauge("filecount", gauge,
map[string]string{
"directory": basedir,
})
if glob.MatchString(basedir) && totalSize >= fc.RecursivePrintSizeB {
gauge := map[string]interface{}{
"count": numFiles,
}
if fc.CountSize {
gauge["size"] = totalSize
}
acc.AddGauge("filecount", gauge,
map[string]string{
"directory": basedir,
})
}

return numFiles, totalSize

}

func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
Expand Down Expand Up @@ -235,11 +239,31 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
return err
}

fc.count(acc, fc.Directory)
for _, globDir := range fc.getDirs() {
glob, err := globpath.Compile(globDir)
if err != nil {
acc.AddError(err)
} else {
fc.count(acc, glob.GetRootDir(), glob)
}
}

return nil
}

func (fc *FileCount) getDirs() []string {
dirs := make([]string, len(fc.Directories))
for i, dir := range fc.Directories {
dirs[i] = dir
}

if fc.Directory != "" {
dirs = append(dirs, fc.Directory)
}

return dirs
}

func NewFileCount() *FileCount {
return &FileCount{
Directory: "",
Expand All @@ -250,7 +274,6 @@ func NewFileCount() *FileCount {
Size: "",
SizeB: int64(0),
MTime: internal.Duration{Duration: 0},
RecursivePrint: false,
RecursivePrintSize: "",
RecursivePrintSizeB: int64(0),
fileFilters: nil,
Expand Down
52 changes: 26 additions & 26 deletions plugins/inputs/filecount/filecount_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,34 @@ func TestNoFilters(t *testing.T) {
matches := []string{"foo", "bar", "baz", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"subdir/qux"}
require.True(t, fileCountEquals(fc, len(matches), 4988))
fileCountEquals(t, fc, len(matches), 4988)
}

func TestNoFiltersOnChildDir(t *testing.T) {
fc := getNoFilterFileCount()
fc.Directories = []string{getTestdataDir() + "/*"}
matches := []string{"subdir/quux", "subdir/quuz", "subdir/qux"}

tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size", int64(446)))
}

func TestNameFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc.Name = "ba*"
matches := []string{"bar", "baz"}
require.True(t, fileCountEquals(fc, len(matches), 0))
fileCountEquals(t, fc, len(matches), 0)
}

func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount()
fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"}
require.True(t, fileCountEquals(fc, len(matches), 4542))
fileCountEquals(t, fc, len(matches), 4542)
}

func TestRegularOnlyFilter(t *testing.T) {
Expand All @@ -41,19 +54,19 @@ func TestRegularOnlyFilter(t *testing.T) {
matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
"subdir/qux"}
require.True(t, fileCountEquals(fc, len(matches), 892))
fileCountEquals(t, fc, len(matches), 892)
}

func TestSizeFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc.Size = "-100B"
matches := []string{"foo", "bar", "baz",
"subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches), 0))
fileCountEquals(t, fc, len(matches), 0)

fc.Size = "100B"
matches = []string{"qux", "subdir/qux"}
require.True(t, fileCountEquals(fc, len(matches), 892))
fileCountEquals(t, fc, len(matches), 892)
}

func TestMTimeFilter(t *testing.T) {
Expand All @@ -69,49 +82,36 @@ func TestMTimeFilter(t *testing.T) {
matches := []string{"foo", "bar", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"subdir/qux"}
require.True(t, fileCountEquals(fc, len(matches), 4988))
fileCountEquals(t, fc, len(matches), 4988)

fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"}
require.True(t, fileCountEquals(fc, len(matches), 0))
}

func TestRecursivePrint(t *testing.T) {
fc := getNoFilterFileCount()
fc.RecursivePrint = true

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
tags := map[string]string{"directory": getTestdataDir() + string(os.PathSeparator) + "subdir"}

require.True(t, acc.HasPoint("filecount", tags, "count", int64(3)))
require.True(t, acc.HasPoint("filecount", tags, "size", int64(446)))
fileCountEquals(t, fc, len(matches), 0)
}

func getNoFilterFileCount() FileCount {
return FileCount{
Directory: getTestdataDir(),
Directories: []string{getTestdataDir() + "/"},
CountSize: true,
Name: "*",
Recursive: true,
RegularOnly: false,
Size: "0B",
MTime: internal.Duration{Duration: 0},
RecursivePrint: false,
RecursivePrintSize: "0B",
fileFilters: nil,
}
}

func getTestdataDir() string {
_, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata/", 1)
return strings.Replace(filename, "filecount_test.go", "testdata", 1)
}

func fileCountEquals(fc FileCount, expectedCount int, expectedSize int) bool {
func fileCountEquals(t *testing.T, fc FileCount, expectedCount int, expectedSize int) {
tags := map[string]string{"directory": getTestdataDir()}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
return acc.HasPoint("filecount", tags, "count", int64(expectedCount)) &&
acc.HasPoint("filecount", tags, "size", int64(expectedSize))
require.True(t, acc.HasPoint("filecount", tags, "count", int64(expectedCount)))
require.True(t, acc.HasPoint("filecount", tags, "size", int64(expectedSize)))
}

0 comments on commit 18373c7

Please sign in to comment.