Skip to content

Commit

Permalink
in_tail: Fix log duplication when follow_inode is true
Browse files Browse the repository at this point in the history
There are 2 causes on this issue:

* Wrong inode is set to TailWatcher when follow_inode is true
* A key (TargetInfo) in @Tails isn't updated for a same path even if new
  one has different inode

Fix #3464

Signed-off-by: Takuro Ashie <ashie@clear-code.com>
  • Loading branch information
ashie committed Jul 24, 2021
1 parent d2867c0 commit 3414c81
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 1 deletion.
10 changes: 9 additions & 1 deletion lib/fluent/plugin/in_tail.rb
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,7 @@ def construct_watcher(target_info)

begin
target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
@tails.delete(target_info)
@tails[target_info] = tw
tw.on_notify
rescue Errno::ENOENT, Errno::EACCES => e
Expand Down Expand Up @@ -491,10 +492,17 @@ def update_watcher(target_info, pe)
new_position_entry = @pf[target_info]

if new_position_entry.read_inode == 0
# When follow_inodes is true, it's not cleaned up by refresh_watcher.
# So it should be unwatched here explicitly.
rotated_tw.unwatched = true
# Make sure to delete old key, it has a different ino while the hash key is same.
@tails.delete(rotated_target_info)
@tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
@tails[new_target_info].on_notify
end
else
# Make sure to delete old key, it has a different ino while the hash key is same.
@tails.delete(rotated_target_info)
@tails[new_target_info] = setup_watcher(new_target_info, pe)
@tails[new_target_info].on_notify
end
Expand Down Expand Up @@ -831,7 +839,7 @@ def on_rotate(stat)
# new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
# don't want to swap state because we need latest read offset in pos file even after rotate_wait
if stat
target_info = TargetInfo.new(@path, stat)
target_info = TargetInfo.new(@path, stat.ino)
@update_watcher.call(target_info, @pe)
end
else
Expand Down
44 changes: 44 additions & 0 deletions test/plugin/test_in_tail.rb
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ def create_target_info(path)
"refresh_interval" => "1s",
"read_from_head" => "true",
"format" => "none",
"rotate_wait" => "1s",
"follow_inodes" => "true"
})
SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
Expand Down Expand Up @@ -1984,6 +1985,49 @@ def test_truncate_file_with_follow_inodes
assert_equal({"message" => "test4"}, events[3][2])
d.instance_shutdown
end

# issue #3464
def test_should_replace_target_info
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
f.puts "test1\n"
}
target_info = create_target_info("#{TMP_DIR}/tail.txt")
inodes = []

config = config_element("ROOT", "", {
"path" => "#{TMP_DIR}/tail.txt*",
"pos_file" => "#{TMP_DIR}/tail.pos",
"tag" => "t1",
"refresh_interval" => "60s",
"read_from_head" => "true",
"format" => "none",
"rotate_wait" => "1s",
"follow_inodes" => "true"
})
d = create_driver(config, false)
d.run(timeout: 5) do
while d.events.size < 1 do
sleep 0.1
end
inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
key.ino
end
assert_equal([target_info.ino], inodes)

cleanup_file("#{TMP_DIR}/tail.txt")
File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}

while d.events.size < 2 do
sleep 0.1
end
inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
key.ino
end
new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
assert_not_equal(target_info.ino, new_target_info.ino)
assert_equal([new_target_info.ino], inodes)
end
end
end

sub_test_case "tail_path" do
Expand Down

0 comments on commit 3414c81

Please sign in to comment.