Skip to content

Commit 592127b

Browse files
authored
HDFS-15520 Use visitor pattern to visit namespace tree (#2203)
1 parent 32895f4 commit 592127b

File tree

11 files changed

+535
-6
lines changed

11 files changed

+535
-6
lines changed

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
3737
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
3838
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
39+
import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
3940
import org.apache.hadoop.hdfs.util.Diff;
4041
import org.apache.hadoop.security.AccessControlException;
4142
import org.apache.hadoop.util.ChunkedArrayList;
@@ -76,7 +77,7 @@ final boolean isRoot() {
7677
}
7778

7879
/** Get the {@link PermissionStatus} */
79-
abstract PermissionStatus getPermissionStatus(int snapshotId);
80+
public abstract PermissionStatus getPermissionStatus(int snapshotId);
8081

8182
/** The same as getPermissionStatus(null). */
8283
final PermissionStatus getPermissionStatus() {
@@ -1123,6 +1124,14 @@ public void clear() {
11231124
}
11241125
}
11251126

1127+
/** Accept a visitor to visit this {@link INode}. */
1128+
public void accept(NamespaceVisitor visitor, int snapshot) {
1129+
final Class<?> clazz = visitor != null? visitor.getClass()
1130+
: NamespaceVisitor.class;
1131+
throw new UnsupportedOperationException(getClass().getSimpleName()
1132+
+ " does not support " + clazz.getSimpleName());
1133+
}
1134+
11261135
/**
11271136
* INode feature such as {@link FileUnderConstructionFeature}
11281137
* and {@link DirectoryWithQuotaFeature}.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.hadoop.hdfs.protocol.SnapshotException;
3535
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
3636
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
37+
import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
3738
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
3839
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
3940
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
@@ -996,6 +997,11 @@ public SnapshotAndINode(int snapshot, INode inode) {
996997
}
997998
}
998999

1000+
@Override
1001+
public void accept(NamespaceVisitor visitor, int snapshot) {
1002+
visitor.visitDirectoryRecursively(this, snapshot);
1003+
}
1004+
9991005
public final int getChildrenNum(final int snapshotId) {
10001006
return getChildrenList(snapshotId).size();
10011007
}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
5454
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
5555
import org.apache.hadoop.hdfs.server.namenode.snapshot.DiffList;
56+
import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
5657
import org.apache.hadoop.hdfs.util.LongBitFormat;
5758
import org.apache.hadoop.util.StringUtils;
5859
import static org.apache.hadoop.io.erasurecode.ErasureCodeConstants.REPLICATION_POLICY_ID;
@@ -1111,6 +1112,11 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
11111112
out.println();
11121113
}
11131114

1115+
@Override
1116+
public void accept(NamespaceVisitor visitor, int snapshot) {
1117+
visitor.visitFile(this, snapshot);
1118+
}
1119+
11141120
/**
11151121
* Remove full blocks at the end file up to newLength
11161122
* @return sum of sizes of the remained blocks

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
3232

3333
import com.google.common.base.Preconditions;
34+
import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
3435
import org.apache.hadoop.security.AccessControlException;
3536

3637
/**
@@ -368,7 +369,12 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
368369
b.append("->");
369370
getReferredINode().dumpTreeRecursively(out, b, snapshot);
370371
}
371-
372+
373+
@Override
374+
public void accept(NamespaceVisitor visitor, int snapshot) {
375+
visitor.visitReferenceRecursively(this, snapshot);
376+
}
377+
372378
public int getDstSnapshotId() {
373379
return Snapshot.CURRENT_STATE_ID;
374380
}
@@ -399,7 +405,7 @@ public WithCount(INodeReference parent, INode referred) {
399405
INodeReferenceValidation.add(this, WithCount.class);
400406
}
401407

402-
private String getCountDetails() {
408+
public String getCountDetails() {
403409
final StringBuilder b = new StringBuilder("[");
404410
if (!withNameList.isEmpty()) {
405411
final Iterator<WithName> i = withNameList.iterator();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.hadoop.hdfs.DFSUtil;
2525
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
2626
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
27+
import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
2728

2829
/**
2930
* An {@link INode} representing a symbolic link.
@@ -104,7 +105,13 @@ public ContentSummaryComputationContext computeContentSummary(int snapshotId,
104105
public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
105106
final int snapshot) {
106107
super.dumpTreeRecursively(out, prefix, snapshot);
107-
out.println();
108+
out.print(" ~> ");
109+
out.println(getSymlinkString());
110+
}
111+
112+
@Override
113+
public void accept(NamespaceVisitor visitor, int snapshot) {
114+
visitor.visitSymlink(this, snapshot);
108115
}
109116

110117
@Override

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ final void clonePermissionStatus(INodeWithAdditionalFields that) {
178178
}
179179

180180
@Override
181-
final PermissionStatus getPermissionStatus(int snapshotId) {
181+
public final PermissionStatus getPermissionStatus(int snapshotId) {
182182
return new PermissionStatus(getUserName(snapshotId), getGroupName(snapshotId),
183183
getFsPermission(snapshotId));
184184
}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ void setSnapshotRoot(INodeDirectoryAttributes root) {
166166
this.isSnapshotRoot = true;
167167
}
168168

169-
boolean isSnapshotRoot() {
169+
public boolean isSnapshotRoot() {
170170
return isSnapshotRoot;
171171
}
172172

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hdfs.server.namenode.visitor;
19+
20+
import com.google.common.base.Preconditions;
21+
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
22+
import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
23+
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
24+
import org.apache.hadoop.hdfs.server.namenode.INode;
25+
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
26+
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
27+
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
28+
import org.apache.hadoop.hdfs.server.namenode.INodeSymlink;
29+
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
30+
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
31+
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
32+
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
33+
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
34+
35+
import java.io.File;
36+
import java.io.FileWriter;
37+
import java.io.IOException;
38+
import java.io.PrintStream;
39+
import java.io.PrintWriter;
40+
import java.io.StringWriter;
41+
42+
/**
43+
* To print the namespace tree recursively for testing.
44+
*
45+
* \- foo (INodeDirectory@33dd2717)
46+
* \- sub1 (INodeDirectory@442172)
47+
* +- file1 (INodeFile@78392d4)
48+
* +- file2 (INodeFile@78392d5)
49+
* +- sub11 (INodeDirectory@8400cff)
50+
* \- file3 (INodeFile@78392d6)
51+
* \- z_file4 (INodeFile@45848712)
52+
*/
53+
public class NamespacePrintVisitor implements NamespaceVisitor {
54+
static final String NON_LAST_ITEM = "+-";
55+
static final String LAST_ITEM = "\\-";
56+
57+
/** Print the tree from the given root to a {@link File}. */
58+
public static void print2File(INode root, File f) throws IOException {
59+
try(final PrintWriter out = new PrintWriter(new FileWriter(f), true)) {
60+
new NamespacePrintVisitor(out).print(root);
61+
}
62+
}
63+
64+
/** @return string of the tree in the given {@link FSNamesystem}. */
65+
public static String print2Sting(FSNamesystem ns) {
66+
return print2Sting(ns.getFSDirectory().getRoot());
67+
}
68+
69+
/** @return string of the tree from the given root. */
70+
public static String print2Sting(INode root) {
71+
final StringWriter out = new StringWriter();
72+
new NamespacePrintVisitor(new PrintWriter(out)).print(root);
73+
return out.getBuffer().toString();
74+
}
75+
76+
/**
77+
* Print the tree in the given {@link FSNamesystem}
78+
* to the given {@link PrintStream}.
79+
*/
80+
public static void print(FSNamesystem ns, PrintStream out) {
81+
new NamespacePrintVisitor(new PrintWriter(out)).print(ns);
82+
}
83+
84+
private final PrintWriter out;
85+
private final StringBuffer prefix = new StringBuffer();
86+
87+
private NamespacePrintVisitor(PrintWriter out) {
88+
this.out = out;
89+
}
90+
91+
private void print(FSNamesystem namesystem) {
92+
print(namesystem.getFSDirectory().getRoot());
93+
}
94+
95+
private void print(INode root) {
96+
root.accept(this, Snapshot.CURRENT_STATE_ID);
97+
}
98+
99+
private void printINode(INode iNode, int snapshot) {
100+
out.print(prefix);
101+
out.print(" ");
102+
final String name = iNode.getLocalName();
103+
out.print(name != null && name.isEmpty()? "/": name);
104+
out.print(" (");
105+
out.print(iNode.getObjectString());
106+
out.print("), ");
107+
out.print(iNode.getParentString());
108+
out.print(", " + iNode.getPermissionStatus(snapshot));
109+
}
110+
111+
@Override
112+
public void visitFile(INodeFile file, int snapshot) {
113+
printINode(file, snapshot);
114+
115+
out.print(", fileSize=" + file.computeFileSize(snapshot));
116+
// print only the first block, if it exists
117+
out.print(", blocks=");
118+
final BlockInfo[] blocks = file.getBlocks();
119+
out.print(blocks.length == 0 ? null: blocks[0]);
120+
out.println();
121+
122+
final FileWithSnapshotFeature snapshotFeature
123+
= file.getFileWithSnapshotFeature();
124+
if (snapshotFeature != null) {
125+
if (prefix.length() >= 2) {
126+
prefix.setLength(prefix.length() - 2);
127+
prefix.append(" ");
128+
}
129+
out.print(prefix);
130+
out.print(snapshotFeature);
131+
}
132+
out.println();
133+
}
134+
135+
@Override
136+
public void visitSymlink(INodeSymlink symlink, int snapshot) {
137+
printINode(symlink, snapshot);
138+
out.print(" ~> ");
139+
out.println(symlink.getSymlinkString());
140+
}
141+
142+
@Override
143+
public void visitReference(INodeReference ref, int snapshot) {
144+
printINode(ref, snapshot);
145+
146+
if (ref instanceof INodeReference.DstReference) {
147+
out.print(", dstSnapshotId=" + ref.getDstSnapshotId());
148+
} else if (ref instanceof INodeReference.WithCount) {
149+
out.print(", " + ((INodeReference.WithCount)ref).getCountDetails());
150+
}
151+
out.println();
152+
}
153+
154+
@Override
155+
public void preVisitReferred(INode referred) {
156+
prefix.setLength(prefix.length() - 2);
157+
prefix.append(" ->");
158+
}
159+
160+
@Override
161+
public void postVisitReferred(INode referred) {
162+
prefix.setLength(prefix.length() - 2);
163+
}
164+
165+
@Override
166+
public void visitDirectory(INodeDirectory dir, int snapshot) {
167+
printINode(dir, snapshot);
168+
169+
out.print(", childrenSize=" + dir.getChildrenList(snapshot).size());
170+
final DirectoryWithQuotaFeature q = dir.getDirectoryWithQuotaFeature();
171+
if (q != null) {
172+
out.print(", " + q);
173+
}
174+
if (dir instanceof Snapshot.Root) {
175+
out.print(", snapshotId=" + snapshot);
176+
}
177+
out.println();
178+
179+
if (prefix.length() >= 2) {
180+
prefix.setLength(prefix.length() - 2);
181+
prefix.append(" ");
182+
}
183+
184+
final DirectoryWithSnapshotFeature snapshotFeature
185+
= dir.getDirectoryWithSnapshotFeature();
186+
if (snapshotFeature != null) {
187+
out.print(prefix);
188+
out.print(snapshotFeature);
189+
}
190+
out.println();
191+
}
192+
193+
@Override
194+
public void visitSnapshottable(INodeDirectory dir,
195+
DirectorySnapshottableFeature snapshottable) {
196+
out.println();
197+
out.print(prefix);
198+
199+
out.print("Snapshot of ");
200+
final String name = dir.getLocalName();
201+
out.print(name != null && name.isEmpty()? "/": name);
202+
out.print(": quota=");
203+
out.print(snapshottable.getSnapshotQuota());
204+
205+
int n = 0;
206+
for(DirectoryDiff diff : snapshottable.getDiffs()) {
207+
if (diff.isSnapshotRoot()) {
208+
n++;
209+
}
210+
}
211+
final int numSnapshots = snapshottable.getNumSnapshots();
212+
Preconditions.checkState(n == numSnapshots,
213+
"numSnapshots = " + numSnapshots + " != " + n);
214+
out.print(", #snapshot=");
215+
out.println(n);
216+
}
217+
218+
@Override
219+
public void preVisitSub(Element sub, int index, boolean isLast) {
220+
prefix.append(isLast? LAST_ITEM : NON_LAST_ITEM);
221+
}
222+
223+
@Override
224+
public void postVisitSub(Element sub, int index, boolean isLast) {
225+
prefix.setLength(prefix.length() - 2);
226+
}
227+
}

0 commit comments

Comments
 (0)