Skip to content

feat!: optimize rename op #300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 26, 2021
Merged

feat!: optimize rename op #300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
37f0c66
fix: update substree
gangliao Feb 25, 2021
e0555cd
feat!: optimize rename op
gangliao Feb 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions ...ject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.util.Map;
import java.util.Queue;
import java.util.LinkedList;
import java.util.Set;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.StorageType;
Expand Down Expand Up @@ -702,8 +703,52 @@ public void localRename(INode node, String oldName, String oldParent, String new
}
}

void update_subtree(Set<INode> renameSet) {
List<Long> longAttr = new ArrayList<>();
List<String> strAttr = new ArrayList<>();

List<Long> fileIds = new ArrayList<>();
List<String> fileAttr = new ArrayList<>();
Iterator<INode> iterator = renameSet.iterator();
while (iterator.hasNext()) {
INode inode = iterator.next();
if (inode == null) continue;
strAttr.add(inode.getLocalName());
if (inode.getId() == 16385) {
strAttr.add(" ");
} else {
strAttr.add(inode.getParentName());
}
longAttr.add(inode.getParentId());
longAttr.add(inode.getId());
longAttr.add(inode.getModificationTime());
longAttr.add(inode.getAccessTime());
longAttr.add(inode.getPermissionLong());
if (inode.isDirectory()) {
longAttr.add(0L);
} else {
longAttr.add(inode.asFile().getHeaderLong());
FileUnderConstructionFeature uc = inode.asFile().getFileUnderConstructionFeature();
if (uc != null) {
fileIds.add(inode.getId());
fileAttr.add(uc.getClientName(inode.getId()));
fileAttr.add(uc.getClientMachine(inode.getId()));
}
}
iterator.remove();
}
try {
if (strAttr.size() > 0) {
DatabaseINode.batchUpdateINodes(longAttr, strAttr, fileIds, fileAttr);
}
} catch (Exception e) {
e.printStackTrace();
}
}

public void remoteRename(INode node, String oldName, String oldParent, String newParent, String address) {
// FIXME: replace NameNode.getId() with 10000 to simplify the ID assignments
newParent = "/nnThroughputBenchmark/rename";
int skip_id = oldParent.length();
Long old_id = node.getId();
if (node.isDirectory()) {
Expand All @@ -716,6 +761,8 @@ public void remoteRename(INode node, String oldName, String oldParent, String ne
.logDelete(null, old_id, node.getModificationTime(), true);

ImmutablePair<String, String> id = null;
Set<INode> renameSet = new HashSet<>();

while ((id = q.poll()) != null) {
INode child = FSDirectory.getInstance().getInode(id.getLeft(), id.getRight());
if (child != null) {
Expand All @@ -731,10 +778,10 @@ public void remoteRename(INode node, String oldName, String oldParent, String ne
}

if (child.getId() != old_id) {
child.setParent(child.getParentId() + 100000);
child.setParentName(newParent + child.getParentName().substring(skip_id));
child.setParent(child.getParentId() + 40000000);
}
child.setId(child.getId() + 100000);
child.setParentName(newParent + child.getParentName() + child.getLocalName());
child.setId(child.getId() + 40000000);

if (child.isDirectory()) {
// log: create new diretory
Expand All @@ -747,13 +794,23 @@ public void remoteRename(INode node, String oldName, String oldParent, String ne
.getEditLog()
.logOpenFile(null, (INodeFile)child, true, true);
}

renameSet.add(child);
if (renameSet.size() >= 4096) {
update_subtree(renameSet);
}
}
}
if (renameSet.size() > 0) {
update_subtree(renameSet);
}

// CompletableFuture.runAsync(() -> {
// stored procedure:
// (1) update subtree IDs and parent fields
DatabaseINode.updateSubtree(old_id, 100000, oldParent, newParent, node.getParentId());
// DatabaseINode.updateSubtree(old_id, 100000, oldParent, "/nnThroughputBenchmark/rename", node.getParentId());
// (2) remove all childs recursively
DatabaseINode.removeChild(old_id);
// }, Database.getInstance().getExecutorService());

// invalidate inode, and childs will be evicted eventually
Expand Down
9 changes: 9 additions & 0 deletions ...t/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeKeyedObjects.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ public static Set<String> getRenameSet() {
}

private static void insertToDB() {
if (concurrentUpdateSet == null) {
return;
}
int i = 0;
final int num = 1024;
long updateSize = concurrentUpdateSet.size();
Expand Down Expand Up @@ -157,6 +160,9 @@ private static void insertToDB() {
}

private static void removeToDB() {
if (concurrentRemoveSet == null) {
return;
}
int i = 0;
final int num = 1024;
List<Long> removeIds = new ArrayList<>();
Expand Down Expand Up @@ -202,6 +208,9 @@ private static void removeToDB() {
}

private static void renameToDB() {
if (concurrentRenameSet == null) {
return;
}
int i = 0;
final int num = 1024;
long renameSize = concurrentRenameSet.size();
Expand Down
46 changes: 21 additions & 25 deletions voltdb/UpdateSubtree.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,21 @@ public long run(final long dir_id, final long dest_id, final String old_parent_n
for (int j = 0; j < res.length; ++j) {
for (int k = 0; k < res[j].getRowCount(); ++k) {
VoltTableRow row = res[j].fetchRow(k);
row.resetRowPosition();
while (row.advanceRow()) {
set.add(row.getLong(0));
}
set.add(row.getLong(0));
}
}
}

for (Long child : set) {
voltQueueSQL(sql2, child);
}
VoltTable[] res = voltExecuteSQL();

for (Long child : set) {
voltQueueSQL(sql4, child);
}
voltExecuteSQL();

Long id = null;
String name = null;
Long accessTime = null;
Expand All @@ -53,27 +56,24 @@ public long run(final long dir_id, final long dest_id, final String old_parent_n
for (int j = 0; j < res.length; ++j) {
for (i = 0; i < res[j].getRowCount(); ++i) {
VoltTableRow row = res[j].fetchRow(i);
row.resetRowPosition();
while (row.advanceRow()) {
id = row.getLong(0);
name = row.getString(1);
accessTime = row.getLong(2);
modificationTime = row.getLong(3);
permission = row.getLong(4);
header = row.getLong(5);
parent = row.getLong(6);
parentName = row.getString(7);
}
id = row.getLong(0);
name = row.getString(1);
accessTime = row.getLong(2);
modificationTime = row.getLong(3);
permission = row.getLong(4);
header = row.getLong(5);
parent = row.getLong(6);
parentName = row.getString(7);

if (id == dir_id) {
id += dest_id;
id += dest_id;
parent = new_parent;
parentName = new_parent_name;
parentName = new_parent_name;
} else {
id += dest_id;
id += dest_id;
parent += dest_id;
parentName = new_parent_name + parentName.substring(old_parent_name.length());
}

voltQueueSQL(sql3,
id,
name,
Expand All @@ -85,12 +85,8 @@ public long run(final long dir_id, final long dest_id, final String old_parent_n
parentName);
}
}
voltExecuteSQL();

for (Long child : set) {
voltQueueSQL(sql4, child);
}
voltExecuteSQL();

return 1;
}
}