Implemented hyrbid watcher for systems without recursive watching.

This commit implements a hybrid watching mechanisms for systems that
don't support native recursive filesystem watching. This is accomplished
by keeping a limited number of watches on only directories with recently
updated contents.

This necessitated vendoring a subset of the github.com/rjeczalik/notify
package (heavily modified to directly expose watcher functionality).

pkg/filesystem/notify/debug.go

@@ -0,0 +1,44 @@
+// Subset of https://github.com/rjeczalik/notify extracted and modified to
+// expose watcher functionality directly. Originally extracted from the
+// following revision:
+// https://github.com/rjeczalik/notify/tree/52ae50d8490436622a8941bd70c3dbe0acdd4bbf
+//
+// The original code license:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2014-2015 The Notify Authors
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// The original license header inside the code itself:
+//
+// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
+// Use of this source code is governed by the MIT license that can be
+// found in the LICENSE file.
+
+package notify
+
+func dbgprint(...interface{}) {}
+
+func dbgprintf(string, ...interface{}) {}
+
+func dbgcallstack(int) []string {
+	return nil
+}

pkg/filesystem/notify/event.go

@@ -0,0 +1,174 @@
+// Subset of https://github.com/rjeczalik/notify extracted and modified to
+// expose watcher functionality directly. Originally extracted from the
+// following revision:
+// https://github.com/rjeczalik/notify/tree/52ae50d8490436622a8941bd70c3dbe0acdd4bbf
+//
+// The original code license:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2014-2015 The Notify Authors
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// The original license header inside the code itself:
+//
+// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
+// Use of this source code is governed by the MIT license that can be
+// found in the LICENSE file.
+
+package notify
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Event represents the type of filesystem action.
+//
+// Number of available event values is dependent on the target system or the
+// watcher implmenetation used (e.g. it's possible to use either kqueue or
+// FSEvents on Darwin).
+//
+// Please consult documentation for your target platform to see list of all
+// available events.
+type Event uint32
+
+// Create, Remove, Write and Rename are the only event values guaranteed to be
+// present on all platforms.
+const (
+	Create = osSpecificCreate
+	Remove = osSpecificRemove
+	Write  = osSpecificWrite
+	Rename = osSpecificRename
+
+	// All is handful alias for all platform-independent event values.
+	All = Create | Remove | Write | Rename
+)
+
+const internal = recursive | omit
+
+// String implements fmt.Stringer interface.
+func (e Event) String() string {
+	var s []string
+	for _, strmap := range []map[Event]string{estr, osestr} {
+		for ev, str := range strmap {
+			if e&ev == ev {
+				s = append(s, str)
+			}
+		}
+	}
+	return strings.Join(s, "|")
+}
+
+// EventInfo describes an event reported by the underlying filesystem notification
+// subsystem.
+//
+// It always describes single event, even if the OS reported a coalesced action.
+// Reported path is absolute and clean.
+//
+// For non-recursive watchpoints its base is always equal to the path passed
+// to corresponding Watch call.
+//
+// The value of Sys if system-dependent and can be nil.
+//
+// Sys
+//
+// Under Darwin (FSEvents) Sys() always returns a non-nil *notify.FSEvent value,
+// which is defined as:
+//
+//   type FSEvent struct {
+//       Path  string // real path of the file or directory
+//       ID    uint64 // ID of the event (FSEventStreamEventId)
+//       Flags uint32 // joint FSEvents* flags (FSEventStreamEventFlags)
+//   }
+//
+// For possible values of Flags see Darwin godoc for notify or FSEvents
+// documentation for FSEventStreamEventFlags constants:
+//
+//    https://developer.apple.com/library/mac/documentation/Darwin/Reference/FSEvents_Ref/index.html#//apple_ref/doc/constant_group/FSEventStreamEventFlags
+//
+// Under Linux (inotify) Sys() always returns a non-nil *unix.InotifyEvent
+// value, defined as:
+//
+//   type InotifyEvent struct {
+//       Wd     int32    // Watch descriptor
+//       Mask   uint32   // Mask describing event
+//       Cookie uint32   // Unique cookie associating related events (for rename(2))
+//       Len    uint32   // Size of name field
+//       Name   [0]uint8 // Optional null-terminated name
+//   }
+//
+// More information about inotify masks and the usage of inotify_event structure
+// can be found at:
+//
+//    http://man7.org/linux/man-pages/man7/inotify.7.html
+//
+// Under Darwin, DragonFlyBSD, FreeBSD, NetBSD, OpenBSD (kqueue) Sys() always
+// returns a non-nil *notify.Kevent value, which is defined as:
+//
+//   type Kevent struct {
+//       Kevent *syscall.Kevent_t // Kevent is a kqueue specific structure
+//       FI     os.FileInfo       // FI describes file/dir
+//   }
+//
+// More information about syscall.Kevent_t can be found at:
+//
+//    https://www.freebsd.org/cgi/man.cgi?query=kqueue
+//
+// Under Windows (ReadDirectoryChangesW) Sys() always returns nil. The documentation
+// of watcher's WinAPI function can be found at:
+//
+//    https://msdn.microsoft.com/en-us/library/windows/desktop/aa365465%28v=vs.85%29.aspx
+type EventInfo interface {
+	Event() Event     // event value for the filesystem action
+	Path() string     // real path of the file or directory
+	Sys() interface{} // underlying data source (can return nil)
+}
+
+type isDirer interface {
+	isDir() (bool, error)
+}
+
+var _ fmt.Stringer = (*event)(nil)
+var _ isDirer = (*event)(nil)
+
+// String implements fmt.Stringer interface.
+func (e *event) String() string {
+	return e.Event().String() + `: "` + e.Path() + `"`
+}
+
+var estr = map[Event]string{
+	Create: "notify.Create",
+	Remove: "notify.Remove",
+	Write:  "notify.Write",
+	Rename: "notify.Rename",
+	// Display name for recursive event is added only for debugging
+	// purposes. It's an internal event after all and won't be exposed to the
+	// user. Having Recursive event printable is helpful, e.g. for reading
+	// testing failure messages:
+	//
+	//    --- FAIL: TestWatchpoint (0.00 seconds)
+	//    watchpoint_test.go:64: want diff=[notify.Remove notify.Create|notify.Remove];
+	//    got [notify.Remove notify.Remove|notify.Create] (i=1)
+	//
+	// Yup, here the diff have Recursive event inside. Go figure.
+	recursive: "recursive",
+	omit:      "omit",
+}

pkg/filesystem/notify/event_inotify.go

@@ -0,0 +1,106 @@
+// Subset of https://github.com/rjeczalik/notify extracted and modified to
+// expose watcher functionality directly. Originally extracted from the
+// following revision:
+// https://github.com/rjeczalik/notify/tree/52ae50d8490436622a8941bd70c3dbe0acdd4bbf
+//
+// The original code license:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2014-2015 The Notify Authors
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// The original license header inside the code itself:
+//
+// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
+// Use of this source code is governed by the MIT license that can be
+// found in the LICENSE file.
+
+// +build linux
+
+package notify
+
+import "golang.org/x/sys/unix"
+
+// Platform independent event values.
+const (
+	osSpecificCreate Event = 0x100000 << iota
+	osSpecificRemove
+	osSpecificWrite
+	osSpecificRename
+	// internal
+	// recursive is used to distinguish recursive eventsets from non-recursive ones
+	recursive
+	// omit is used for dispatching internal events; only those events are sent
+	// for which both the event and the watchpoint has omit in theirs event sets.
+	omit
+)
+
+// Inotify specific masks are legal, implemented events that are guaranteed to
+// work with notify package on linux-based systems.
+const (
+	InAccess       = Event(unix.IN_ACCESS)        // File was accessed
+	InModify       = Event(unix.IN_MODIFY)        // File was modified
+	InAttrib       = Event(unix.IN_ATTRIB)        // Metadata changed
+	InCloseWrite   = Event(unix.IN_CLOSE_WRITE)   // Writtable file was closed
+	InCloseNowrite = Event(unix.IN_CLOSE_NOWRITE) // Unwrittable file closed
+	InOpen         = Event(unix.IN_OPEN)          // File was opened
+	InMovedFrom    = Event(unix.IN_MOVED_FROM)    // File was moved from X
+	InMovedTo      = Event(unix.IN_MOVED_TO)      // File was moved to Y
+	InCreate       = Event(unix.IN_CREATE)        // Subfile was created
+	InDelete       = Event(unix.IN_DELETE)        // Subfile was deleted
+	InDeleteSelf   = Event(unix.IN_DELETE_SELF)   // Self was deleted
+	InMoveSelf     = Event(unix.IN_MOVE_SELF)     // Self was moved
+)
+
+var osestr = map[Event]string{
+	InAccess:       "notify.InAccess",
+	InModify:       "notify.InModify",
+	InAttrib:       "notify.InAttrib",
+	InCloseWrite:   "notify.InCloseWrite",
+	InCloseNowrite: "notify.InCloseNowrite",
+	InOpen:         "notify.InOpen",
+	InMovedFrom:    "notify.InMovedFrom",
+	InMovedTo:      "notify.InMovedTo",
+	InCreate:       "notify.InCreate",
+	InDelete:       "notify.InDelete",
+	InDeleteSelf:   "notify.InDeleteSelf",
+	InMoveSelf:     "notify.InMoveSelf",
+}
+
+// Inotify behavior events are not **currently** supported by notify package.
+const (
+	inDontFollow = Event(unix.IN_DONT_FOLLOW)
+	inExclUnlink = Event(unix.IN_EXCL_UNLINK)
+	inMaskAdd    = Event(unix.IN_MASK_ADD)
+	inOneshot    = Event(unix.IN_ONESHOT)
+	inOnlydir    = Event(unix.IN_ONLYDIR)
+)
+
+type event struct {
+	sys   unix.InotifyEvent
+	path  string
+	event Event
+}
+
+func (e *event) Event() Event         { return e.event }
+func (e *event) Path() string         { return e.path }
+func (e *event) Sys() interface{}     { return &e.sys }
+func (e *event) isDir() (bool, error) { return e.sys.Mask&unix.IN_ISDIR != 0, nil }