Skip to content

Commit 8370a0a

Browse files
Adam Antaljojochuang
Adam Antal
authored and
jojochuang
committed
HDFS-14203. Refactor OIV Delimited output entry building mechanism. Contributed by Adam Antal.
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
1 parent 0766186 commit 8370a0a

File tree

2 files changed

+203
-65
lines changed

2 files changed

+203
-65
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java‎

Lines changed: 91 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import java.io.IOException;
2828
import java.io.PrintStream;
2929
import java.text.SimpleDateFormat;
30-
import java.util.Date;
3130

3231
/**
3332
* A PBImageDelimitedTextWriter generates a text representation of the PB fsimage,
@@ -44,80 +43,108 @@
4443
* constructor.
4544
*/
4645
public class PBImageDelimitedTextWriter extends PBImageTextWriter {
47-
private static final String DATE_FORMAT="yyyy-MM-dd HH:mm";
48-
private final SimpleDateFormat dateFormatter =
49-
new SimpleDateFormat(DATE_FORMAT);
46+
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm";
47+
48+
static class OutputEntryBuilder {
49+
private final SimpleDateFormat dateFormatter =
50+
new SimpleDateFormat(DATE_FORMAT);
51+
52+
private PBImageDelimitedTextWriter writer;
53+
private Path path;
54+
private int replication = 0;
55+
private long modificationTime;
56+
private long accessTime = 0;
57+
private long preferredBlockSize = 0;
58+
private int blocksCount = 0;
59+
private long fileSize = 0;
60+
private long nsQuota = 0;
61+
private long dsQuota = 0;
62+
63+
private String dirPermission = "-";
64+
private PermissionStatus permissionStatus;
65+
private String aclPermission = "";
66+
67+
OutputEntryBuilder(PBImageDelimitedTextWriter writer, INode inode) {
68+
this.writer = writer;
69+
switch (inode.getType()) {
70+
case FILE:
71+
INodeFile file = inode.getFile();
72+
replication = file.getReplication();
73+
modificationTime = file.getModificationTime();
74+
accessTime = file.getAccessTime();
75+
preferredBlockSize = file.getPreferredBlockSize();
76+
blocksCount = file.getBlocksCount();
77+
fileSize = FSImageLoader.getFileSize(file);
78+
permissionStatus = writer.getPermission(file.getPermission());
79+
if (file.hasAcl() && file.getAcl().getEntriesCount() > 0){
80+
aclPermission = "+";
81+
}
82+
break;
83+
case DIRECTORY:
84+
INodeDirectory dir = inode.getDirectory();
85+
modificationTime = dir.getModificationTime();
86+
nsQuota = dir.getNsQuota();
87+
dsQuota = dir.getDsQuota();
88+
dirPermission = "d";
89+
permissionStatus = writer.getPermission(dir.getPermission());
90+
if (dir.hasAcl() && dir.getAcl().getEntriesCount() > 0){
91+
aclPermission = "+";
92+
}
93+
break;
94+
case SYMLINK:
95+
INodeSymlink s = inode.getSymlink();
96+
modificationTime = s.getModificationTime();
97+
accessTime = s.getAccessTime();
98+
permissionStatus = writer.getPermission(s.getPermission());
99+
break;
100+
default:
101+
break;
102+
}
103+
}
104+
105+
void setPath(Path path) {
106+
this.path = path;
107+
}
108+
109+
public String build() {
110+
assert permissionStatus != null : "The PermissionStatus is null!";
111+
assert permissionStatus.getUserName() != null : "User name is null!";
112+
assert permissionStatus.getGroupName() != null : "Group name is null!";
113+
114+
StringBuffer buffer = new StringBuffer();
115+
writer.append(buffer, path.toString());
116+
writer.append(buffer, replication);
117+
writer.append(buffer, dateFormatter.format(modificationTime));
118+
writer.append(buffer, dateFormatter.format(accessTime));
119+
writer.append(buffer, preferredBlockSize);
120+
writer.append(buffer, blocksCount);
121+
writer.append(buffer, fileSize);
122+
writer.append(buffer, nsQuota);
123+
writer.append(buffer, dsQuota);
124+
writer.append(buffer, dirPermission +
125+
permissionStatus.getPermission().toString() + aclPermission);
126+
writer.append(buffer, permissionStatus.getUserName());
127+
writer.append(buffer, permissionStatus.getGroupName());
128+
return buffer.substring(1);
129+
}
130+
}
50131

51132
PBImageDelimitedTextWriter(PrintStream out, String delimiter, String tempPath)
52133
throws IOException {
53134
super(out, delimiter, tempPath);
54135
}
55136

56-
private String formatDate(long date) {
57-
return dateFormatter.format(new Date(date));
58-
}
59-
60137
@Override
61138
public String getEntry(String parent, INode inode) {
62-
StringBuffer buffer = new StringBuffer();
139+
OutputEntryBuilder entryBuilder =
140+
new OutputEntryBuilder(this, inode);
141+
63142
String inodeName = inode.getName().toStringUtf8();
64143
Path path = new Path(parent.isEmpty() ? "/" : parent,
65144
inodeName.isEmpty() ? "/" : inodeName);
66-
append(buffer, path.toString());
67-
PermissionStatus p = null;
68-
boolean isDir = false;
69-
boolean hasAcl = false;
70-
71-
switch (inode.getType()) {
72-
case FILE:
73-
INodeFile file = inode.getFile();
74-
p = getPermission(file.getPermission());
75-
hasAcl = file.hasAcl() && file.getAcl().getEntriesCount() > 0;
76-
append(buffer, file.getReplication());
77-
append(buffer, formatDate(file.getModificationTime()));
78-
append(buffer, formatDate(file.getAccessTime()));
79-
append(buffer, file.getPreferredBlockSize());
80-
append(buffer, file.getBlocksCount());
81-
append(buffer, FSImageLoader.getFileSize(file));
82-
append(buffer, 0); // NS_QUOTA
83-
append(buffer, 0); // DS_QUOTA
84-
break;
85-
case DIRECTORY:
86-
INodeDirectory dir = inode.getDirectory();
87-
p = getPermission(dir.getPermission());
88-
hasAcl = dir.hasAcl() && dir.getAcl().getEntriesCount() > 0;
89-
append(buffer, 0); // Replication
90-
append(buffer, formatDate(dir.getModificationTime()));
91-
append(buffer, formatDate(0)); // Access time.
92-
append(buffer, 0); // Block size.
93-
append(buffer, 0); // Num blocks.
94-
append(buffer, 0); // Num bytes.
95-
append(buffer, dir.getNsQuota());
96-
append(buffer, dir.getDsQuota());
97-
isDir = true;
98-
break;
99-
case SYMLINK:
100-
INodeSymlink s = inode.getSymlink();
101-
p = getPermission(s.getPermission());
102-
append(buffer, 0); // Replication
103-
append(buffer, formatDate(s.getModificationTime()));
104-
append(buffer, formatDate(s.getAccessTime()));
105-
append(buffer, 0); // Block size.
106-
append(buffer, 0); // Num blocks.
107-
append(buffer, 0); // Num bytes.
108-
append(buffer, 0); // NS_QUOTA
109-
append(buffer, 0); // DS_QUOTA
110-
break;
111-
default:
112-
break;
113-
}
114-
assert p != null;
115-
String dirString = isDir ? "d" : "-";
116-
String aclString = hasAcl ? "+" : "";
117-
append(buffer, dirString + p.getPermission().toString() + aclString);
118-
append(buffer, p.getUserName());
119-
append(buffer, p.getGroupName());
120-
return buffer.substring(1);
145+
entryBuilder.setPath(path);
146+
147+
return entryBuilder.build();
121148
}
122149

123150
@Override

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java‎

Lines changed: 112 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@
4040
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA;
4141
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME;
4242
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_OPTION;
43+
import static org.mockito.ArgumentMatchers.anyLong;
44+
import static org.mockito.Mockito.spy;
45+
import static org.mockito.Mockito.when;
4346
import org.apache.hadoop.io.erasurecode.ECSchema;
4447
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
4548
import static org.junit.Assert.assertEquals;
@@ -98,14 +101,17 @@
98101
import org.apache.hadoop.fs.Path;
99102
import org.apache.hadoop.fs.permission.FsAction;
100103
import org.apache.hadoop.fs.permission.FsPermission;
104+
import org.apache.hadoop.fs.permission.PermissionStatus;
101105
import org.apache.hadoop.hdfs.DFSConfigKeys;
102106
import org.apache.hadoop.hdfs.DFSTestUtil;
103107
import org.apache.hadoop.hdfs.DistributedFileSystem;
104108
import org.apache.hadoop.hdfs.MiniDFSCluster;
105109
import org.apache.hadoop.hdfs.protocol.BlockType;
106110
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
107111
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
112+
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
108113
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
114+
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
109115
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
110116
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
111117
import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
@@ -132,6 +138,7 @@
132138

133139
import com.google.common.collect.Lists;
134140
import com.google.common.collect.Maps;
141+
import com.google.protobuf.ByteString;
135142

136143
public class TestOfflineImageViewer {
137144
private static final Logger LOG =
@@ -146,6 +153,7 @@ public class TestOfflineImageViewer {
146153
private static final long FILE_NODE_ID_2 = 16389;
147154
private static final long FILE_NODE_ID_3 = 16394;
148155
private static final long DIR_NODE_ID = 16391;
156+
private static final long SAMPLE_TIMESTAMP = 946684800000L;
149157

150158
// namespace as written to dfs, to be compared with viewer's output
151159
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
@@ -659,6 +667,109 @@ public void testWebImageViewerSecureMode() throws Exception {
659667
}
660668
}
661669

670+
private FsImageProto.INodeSection.INode createSampleFileInode() {
671+
HdfsProtos.BlockProto.Builder block =
672+
HdfsProtos.BlockProto.newBuilder()
673+
.setNumBytes(1024)
674+
.setBlockId(8)
675+
.setGenStamp(SAMPLE_TIMESTAMP);
676+
FsImageProto.INodeSection.AclFeatureProto.Builder acl =
677+
FsImageProto.INodeSection.AclFeatureProto.newBuilder()
678+
.addEntries(2);
679+
FsImageProto.INodeSection.INodeFile.Builder file =
680+
FsImageProto.INodeSection.INodeFile.newBuilder()
681+
.setReplication(5)
682+
.setModificationTime(SAMPLE_TIMESTAMP)
683+
.setAccessTime(SAMPLE_TIMESTAMP)
684+
.setPreferredBlockSize(1024)
685+
.addBlocks(block)
686+
.addBlocks(block)
687+
.addBlocks(block)
688+
.setAcl(acl);
689+
690+
return FsImageProto.INodeSection.INode.newBuilder()
691+
.setType(FsImageProto.INodeSection.INode.Type.FILE)
692+
.setFile(file)
693+
.setName(ByteString.copyFromUtf8("file"))
694+
.setId(3)
695+
.build();
696+
}
697+
698+
private FsImageProto.INodeSection.INode createSampleDirInode() {
699+
FsImageProto.INodeSection.AclFeatureProto.Builder acl =
700+
FsImageProto.INodeSection.AclFeatureProto.newBuilder()
701+
.addEntries(2);
702+
FsImageProto.INodeSection.INodeDirectory.Builder directory =
703+
FsImageProto.INodeSection.INodeDirectory.newBuilder()
704+
.setDsQuota(1000)
705+
.setNsQuota(700)
706+
.setModificationTime(SAMPLE_TIMESTAMP)
707+
.setAcl(acl);
708+
709+
return FsImageProto.INodeSection.INode.newBuilder()
710+
.setType(FsImageProto.INodeSection.INode.Type.DIRECTORY)
711+
.setDirectory(directory)
712+
.setName(ByteString.copyFromUtf8("dir"))
713+
.setId(3)
714+
.build();
715+
}
716+
717+
private FsImageProto.INodeSection.INode createSampleSymlink() {
718+
FsImageProto.INodeSection.INodeSymlink.Builder symlink =
719+
FsImageProto.INodeSection.INodeSymlink.newBuilder()
720+
.setModificationTime(SAMPLE_TIMESTAMP)
721+
.setAccessTime(SAMPLE_TIMESTAMP);
722+
723+
return FsImageProto.INodeSection.INode.newBuilder()
724+
.setType(FsImageProto.INodeSection.INode.Type.SYMLINK)
725+
.setSymlink(symlink)
726+
.setName(ByteString.copyFromUtf8("sym"))
727+
.setId(5)
728+
.build();
729+
}
730+
731+
private PBImageDelimitedTextWriter createDelimitedWriterSpy()
732+
throws IOException {
733+
FsPermission fsPermission = new FsPermission(
734+
FsAction.ALL,
735+
FsAction.WRITE_EXECUTE,
736+
FsAction.WRITE);
737+
PermissionStatus permStatus = new PermissionStatus(
738+
"user_1",
739+
"group_1",
740+
fsPermission);
741+
742+
PBImageDelimitedTextWriter writer = new
743+
PBImageDelimitedTextWriter(null, ",", "");
744+
PBImageDelimitedTextWriter writerSpy = spy(writer);
745+
when(writerSpy.getPermission(anyLong())).thenReturn(permStatus);
746+
return writerSpy;
747+
}
748+
749+
@Test
750+
public void testWriterOutputEntryBuilderForFile() throws IOException {
751+
assertEquals("/path/file,5,2000-01-01 00:00,2000-01-01 00:00," +
752+
"1024,3,3072,0,0,-rwx-wx-w-+,user_1,group_1",
753+
createDelimitedWriterSpy().getEntry("/path/",
754+
createSampleFileInode()));
755+
}
756+
757+
@Test
758+
public void testWriterOutputEntryBuilderForDirectory() throws IOException {
759+
assertEquals("/path/dir,0,2000-01-01 00:00,1970-01-01 00:00" +
760+
",0,0,0,700,1000,drwx-wx-w-+,user_1,group_1",
761+
createDelimitedWriterSpy().getEntry("/path/",
762+
createSampleDirInode()));
763+
}
764+
765+
@Test
766+
public void testWriterOutputEntryBuilderForSymlink() throws IOException {
767+
assertEquals("/path/sym,0,2000-01-01 00:00,2000-01-01 00:00" +
768+
",0,0,0,0,0,-rwx-wx-w-,user_1,group_1",
769+
createDelimitedWriterSpy().getEntry("/path/",
770+
createSampleSymlink()));
771+
}
772+
662773
@Test
663774
public void testPBDelimitedWriter() throws IOException, InterruptedException {
664775
testPBDelimitedWriter(""); // Test in memory db.
@@ -667,7 +778,7 @@ public void testPBDelimitedWriter() throws IOException, InterruptedException {
667778
}
668779

669780
@Test
670-
public void testOutputEntryBuilder() throws IOException {
781+
public void testCorruptionOutputEntryBuilder() throws IOException {
671782
PBImageCorruptionDetector corrDetector =
672783
new PBImageCorruptionDetector(null, ",", "");
673784
PBImageCorruption c1 = new PBImageCorruption(342, true, false, 3);

0 commit comments

Comments
 (0)