diff --git a/src/color.c b/src/color.c
index d1b36f96..a7018130 100644
--- a/src/color.c
+++ b/src/color.c
@@ -998,22 +998,25 @@ static ssize_t first_broken_offset(const char *path, const struct BFTW *ftwbuf,
 		goto out;
 	}
 
+	size_t len = dstrlen(at_path);
 	while (ret > 0) {
+		dstresize(&at_path, len);
 		if (xfaccessat(at_fd, at_path, F_OK) == 0) {
 			break;
 		}
 
-		size_t len = dstrlen(at_path);
-		while (ret && at_path[len - 1] == '/') {
-			--len, --ret;
-		}
-		if (errno != ENOTDIR) {
-			while (ret && at_path[len - 1] != '/') {
+		// Try without trailing slashes, to distinguish "notdir/" from "notdir"
+		if (at_path[len - 1] == '/') {
+			do {
 				--len, --ret;
-			}
+			} while (ret > 0 && at_path[len - 1] == '/');
+			continue;
 		}
 
-		dstresize(&at_path, len);
+		// Remove the last component and try again
+		do {
+			--len, --ret;
+		} while (ret > 0 && at_path[len - 1] != '/');
 	}
 
 out_path:
diff --git a/tests/bfs/color_notdir_slash_error.out b/tests/bfs/color_notdir_slash_error.out
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/bfs/color_notdir_slash_error.sh b/tests/bfs/color_notdir_slash_error.sh
new file mode 100644
index 00000000..ca26d504
--- /dev/null
+++ b/tests/bfs/color_notdir_slash_error.sh
@@ -0,0 +1,2 @@
+# Regression test: infinite loop printing the error message for .../notdir/nowhere
+! bfs_diff -color links/notdir/nowhere