Change hpcloud tailer library to poll mode to workaround bugs in the …

…event driven code within the library

Had to rework tests with some waits for messages because polling is slower
There are several longer sleeps in the promtail_test which are not ideal and have been marked with FIXME to remove once we have a better solution

pkg/promtail/promtail_test.go

@@ -179,6 +179,10 @@ func fileRoll(t *testing.T, filename string, prefix string) int {
 		}
 		time.Sleep(1 * time.Millisecond)
 	}
+
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	if err = os.Rename(filename, filename+".1"); err != nil {
 		t.Fatal("Failed to rename file for test: ", err)
 	}
@@ -195,6 +199,9 @@ func fileRoll(t *testing.T, filename string, prefix string) int {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	return 200
 }
 
@@ -224,6 +231,9 @@ func symlinkRoll(t *testing.T, testDir string, filename string, prefix string) i
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	// Remove the link, make a new file, link to the new file.
 	if err := os.Remove(filename); err != nil {
 		t.Fatal(err)
@@ -245,6 +255,9 @@ func symlinkRoll(t *testing.T, testDir string, filename string, prefix string) i
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	return 200
 
 }

pkg/promtail/targets/filetarget_test.go

@@ -68,6 +68,12 @@ func TestLongPositionsSyncDelayStillSavesCorrectPosition(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -155,6 +161,12 @@ func TestWatchEntireDirectory(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -238,6 +250,12 @@ func TestFileRolls(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	// Rename the log file to something not in the pattern, then create a new file with the same name.
 	err = os.Rename(logFile, dirName+"/test.log.1")
 	if err != nil {
@@ -256,6 +274,12 @@ func TestFileRolls(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown = 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	positions.Stop()
 
@@ -324,6 +348,12 @@ func TestResumesWhereLeftOff(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -352,6 +382,12 @@ func TestResumesWhereLeftOff(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown = 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target2.Stop()
 	ps2.Stop()
 
@@ -431,6 +467,12 @@ func TestGlobWithMultipleFiles(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 

pkg/promtail/targets/tailer.go

@@ -48,6 +48,7 @@ func newTailer(logger log.Logger, handler api.EntryHandler, positions *positions
 
 	tail, err := tail.TailFile(filename, tail.Config{
 		Follow: true,
+		Poll:   true,
 		ReOpen: reOpen,
 		Location: &tail.SeekInfo{
 			Offset: positions.Get(filename),