Skip to content

update local rename #303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 28, 2021
Merged

update local rename #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8ddea41
feat!: direct transfer new derived subtree to remote namenode
gangliao Feb 27, 2021
ff48622
feat!: register rename proto
gangliao Feb 27, 2021
6991f41
fix!: add parentname in protobuf
gangliao Feb 27, 2021
60b3a5b
fix: cache lookup
gangliao Feb 27, 2021
aaf1021
update local rename
gangliao Feb 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions ...s-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -866,12 +866,14 @@ public void remoteLogOpenFile(INodeFile newNode, String nameNodeAddress) {
.setId(newNode.getId())
.setName(ByteString.copyFrom(newNode.getLocalNameBytes()))
.setType(INodeSection.INode.Type.FILE).setFile(b)
.setParent(newNode.getParentId()).build();
.setParent(newNode.getParentId())
.setParentName(newNode.getParentName())
.build();

byte[] data = r.toByteArray();
FSEditLogProtocol proxy = (FSEditLogProtocol) RPC.getProxy(
FSEditLogProtocol.class, FSEditLogProtocol.versionID,
new InetSocketAddress(nameNodeAddress, 10086), new Configuration());
new InetSocketAddress(nameNodeAddress, 10087), new Configuration());
proxy.logEdit(data);
} catch (Exception e) {
e.printStackTrace();
Expand Down Expand Up @@ -974,13 +976,15 @@ public void remoteLogMkDir(INodeDirectory newNode, String nameNodeAddress) {
.setId(newNode.getId())
.setName(ByteString.copyFrom(newNode.getLocalNameBytes()))
.setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b)
.setParent(newNode.getParentId()).build();
.setParent(newNode.getParentId())
.setParentName(newNode.getParentName())
.build();

byte[] data = r.toByteArray();

FSEditLogProtocol proxy = (FSEditLogProtocol) RPC.getProxy(
FSEditLogProtocol.class, FSEditLogProtocol.versionID,
new InetSocketAddress(nameNodeAddress, 10086), new Configuration());
new InetSocketAddress(nameNodeAddress, 10087), new Configuration());
proxy.logEdit(data);
} catch (Exception e) {
e.printStackTrace();
Expand Down
46 changes: 41 additions & 5 deletions ...doop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogProtocolImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NamespaceSubtree;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCompactProto;
Expand Down Expand Up @@ -102,7 +103,7 @@ public INodeFile loadINodeFile(INodeSection.INode n) {
final INodeFile file = new INodeFile(n.getId(),
n.getName().toByteArray(), permissions, f.getModificationTime(),
f.getAccessTime(), blocks, replication, ecPolicyID,
f.getPreferredBlockSize(), (byte)f.getStoragePolicyID(), blockType, null);
f.getPreferredBlockSize(), (byte)f.getStoragePolicyID(), blockType, n.getParentName());

if (f.hasAcl()) {
int[] entries = AclEntryStatusFormat.toInt(loadAclEntries(f.getAcl(), null));
Expand Down Expand Up @@ -135,7 +136,11 @@ public INodeFile loadINodeFile(INodeSection.INode n) {
}

// set parent
file.setParent(n.getParent());
INode parent = INodeKeyedObjects.getCache().getIfPresent(file.getParentName());
if (parent != null) {
parent.asDirectory().addChild(file);
// parent.asDirectory().filter.put(String.valueOf(dir.getParentId()) + dirname);
}
return file;
}

Expand All @@ -146,7 +151,7 @@ public INodeDirectory loadINodeDirectory(INodeSection.INode n) {

final PermissionStatus permissions = loadPermission(d.getPermission(), null);
final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName()
.toByteArray(), permissions, d.getModificationTime(), null);
.toByteArray(), permissions, d.getModificationTime(), n.getParentName());
final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();

if (d.hasAcl()) {
Expand All @@ -158,13 +163,44 @@ public INodeDirectory loadINodeDirectory(INodeSection.INode n) {
}

// set parent
dir.setParent(n.getParent());
INode parent = INodeKeyedObjects.getCache().getIfPresent(dir.getParentName());
if (parent != null) {
parent.asDirectory().addChild(dir);
// parent.asDirectory().filter.put(String.valueOf(file.getParentId()) + filename);
}
return dir;
}

@Override
public void logEdit(byte[] in) throws IOException {
return;
NamespaceSubtree tree = null;
try {
tree = NamespaceSubtree.parseFrom(in);
} catch (InvalidProtocolBufferException e) {
e.printStackTrace();
}

for (INodeSection.INode inode : tree.getInodesList()) {
INode parent;
switch (inode.getType()) {
case FILE:
INodeFile file = loadINodeFile(inode);
String filename = file.getLocalName();
INodeKeyedObjects.getCache().put(file.getPath(), file);
FSDirectory.getInstance().getEditLog().logOpenFile(null, file, true, false);
// INodeKeyedObjects.getUpdateSet().add(file.getPath());
break;
case DIRECTORY:
INodeDirectory dir = loadINodeDirectory(inode);
String dirname = DFSUtil.bytes2String(dir.getLocalNameBytes());
INodeKeyedObjects.getCache().put(dir.getPath(), dir);
FSDirectory.getInstance().getEditLog().logMkDir(null, dir);
// INodeKeyedObjects.getUpdateSet().add(inode.getPath());
break;
default:
break;
}
}
}

@Override
Expand Down
Loading