Skip to content

Commit 54f2106

Browse files
abhradeepkunduabhradeepkundu
authored
HBASE-27265 : Tool to read StoreFileTrackerFile (#4673)
Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org> Signed-off-by: Duo Zhang <zhangduo@apache.org>
1 parent 4ebf719 commit 54f2106

File tree

5 files changed

+410
-3
lines changed

5 files changed

+410
-3
lines changed

‎bin/hbase

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ show_usage() {
8383
if [ "${in_omnibus_tarball}" = "true" ]; then
8484
echo " wal Write-ahead-log analyzer"
8585
echo " hfile Store file analyzer"
86+
echo " sft Store file tracker viewer"
8687
echo " zkcli Run the ZooKeeper shell"
8788
echo " master Run an HBase HMaster node"
8889
echo " regionserver Run an HBase HRegionServer node"
@@ -597,6 +598,8 @@ elif [ "$COMMAND" = "wal" ] ; then
597598
CLASS='org.apache.hadoop.hbase.wal.WALPrettyPrinter'
598599
elif [ "$COMMAND" = "hfile" ] ; then
599600
CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter'
601+
elif [ "$COMMAND" = "sft" ] ; then
602+
CLASS='org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileListFilePrettyPrinter'
600603
elif [ "$COMMAND" = "zkcli" ] ; then
601604
CLASS="org.apache.hadoop.hbase.zookeeper.ZKMainServer"
602605
for f in $HBASE_HOME/lib/zkcli/*.jar; do

‎bin/hbase.cmd

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,10 @@ goto :eof
435435
set CLASS=org.apache.hadoop.hbase.io.hfile.HFile
436436
goto :eof
437437

438+
:sft
439+
set CLASS=org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileListFilePrettyPrinter
440+
goto :eof
441+
438442
:zkcli
439443
set CLASS=org.apache.hadoop.hbase.zookeeper.ZKMainServer
440444
set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\lib\zkcli\*
@@ -468,6 +472,7 @@ goto :eof
468472
echo hbck Run the hbase 'fsck' tool
469473
echo wal Write-ahead-log analyzer
470474
echo hfile Store file analyzer
475+
echo sft Store file tracker viewer
471476
echo zkcli Run the ZooKeeper shell
472477
echo master Run an HBase HMaster node
473478
echo regionserver Run an HBase HRegionServer node

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class StoreFileListFile {
7373

7474
private static final char TRACK_FILE_SEPARATOR = '.';
7575

76-
private static final Pattern TRACK_FILE_PATTERN = Pattern.compile("^f(1|2)\\.\\d+$");
76+
static final Pattern TRACK_FILE_PATTERN = Pattern.compile("^f(1|2)\\.\\d+$");
7777

7878
// 16 MB, which is big enough for a tracker file
7979
private static final int MAX_FILE_SIZE = 16 * 1024 * 1024;
@@ -94,8 +94,7 @@ class StoreFileListFile {
9494
trackFileDir = new Path(ctx.getFamilyStoreDirectoryPath(), TRACK_FILE_DIR);
9595
}
9696

97-
private StoreFileList load(Path path) throws IOException {
98-
FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
97+
static StoreFileList load(FileSystem fs, Path path) throws IOException {
9998
byte[] data;
10099
int expectedChecksum;
101100
try (FSDataInputStream in = fs.open(path)) {
@@ -118,6 +117,11 @@ private StoreFileList load(Path path) throws IOException {
118117
return StoreFileList.parseFrom(data);
119118
}
120119

120+
StoreFileList load(Path path) throws IOException {
121+
FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
122+
return load(fs, path);
123+
}
124+
121125
private int select(StoreFileList[] lists) {
122126
if (lists[0] == null) {
123127
return 1;

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFilePrettyPrinter.java

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.regionserver.storefiletracker;
19+
20+
import java.io.IOException;
21+
import java.io.PrintStream;
22+
import org.apache.commons.lang3.StringUtils;
23+
import org.apache.hadoop.conf.Configuration;
24+
import org.apache.hadoop.conf.Configured;
25+
import org.apache.hadoop.fs.FileSystem;
26+
import org.apache.hadoop.fs.LocatedFileStatus;
27+
import org.apache.hadoop.fs.Path;
28+
import org.apache.hadoop.fs.RemoteIterator;
29+
import org.apache.hadoop.hbase.HBaseConfiguration;
30+
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
31+
import org.apache.hadoop.hbase.HConstants;
32+
import org.apache.hadoop.hbase.TableName;
33+
import org.apache.hadoop.hbase.util.CommonFSUtils;
34+
import org.apache.hadoop.util.Tool;
35+
import org.apache.hadoop.util.ToolRunner;
36+
import org.apache.yetus.audience.InterfaceAudience;
37+
import org.apache.yetus.audience.InterfaceStability;
38+
import org.slf4j.Logger;
39+
import org.slf4j.LoggerFactory;
40+
41+
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
42+
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
43+
import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
44+
import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
45+
import org.apache.hbase.thirdparty.org.apache.commons.cli.OptionGroup;
46+
import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
47+
import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
48+
import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
49+
50+
import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
51+
52+
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
53+
@InterfaceStability.Evolving
54+
public class StoreFileListFilePrettyPrinter extends Configured implements Tool {
55+
private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFilePrettyPrinter.class);
56+
57+
private Options options = new Options();
58+
59+
private final String fileOption = "f";
60+
private final String columnFamilyOption = "cf";
61+
private final String regionOption = "r";
62+
private final String tableNameOption = "t";
63+
64+
private String namespace;
65+
private String regionName;
66+
private String columnFamily;
67+
private String tableName;
68+
private Path path;
69+
private PrintStream err = System.err;
70+
private PrintStream out = System.out;
71+
72+
public StoreFileListFilePrettyPrinter() {
73+
super();
74+
init();
75+
}
76+
77+
public StoreFileListFilePrettyPrinter(Configuration conf) {
78+
super(conf);
79+
init();
80+
}
81+
82+
private void init() {
83+
OptionGroup files = new OptionGroup();
84+
options.addOption(new Option(tableNameOption, "table", true,
85+
"Table to scan. Pass table name; e.g. test_table"));
86+
options.addOption(new Option(columnFamilyOption, "columnfamily", true,
87+
"column family to scan. Pass column family name; e.g. f"));
88+
files.addOption(new Option(regionOption, "region", true,
89+
"Region to scan. Pass region name; e.g. '3d58e9067bf23e378e68c071f3dd39eb'"));
90+
files.addOption(new Option(fileOption, "file", true,
91+
"File to scan. Pass full-path; e.g. /root/hbase-3.0.0-alpha-4-SNAPSHOT/hbase-data/"
92+
+ "data/default/tbl-sft/093fa06bf84b3b631007f951a14b8457/f/.filelist/f2.1655139542249"));
93+
options.addOptionGroup(files);
94+
}
95+
96+
public boolean parseOptions(String[] args) throws ParseException, IOException {
97+
HelpFormatter formatter = new HelpFormatter();
98+
if (args.length == 0) {
99+
formatter
100+
.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename>"
101+
+ " --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
102+
return false;
103+
}
104+
105+
CommandLineParser parser = new PosixParser();
106+
CommandLine cmd = parser.parse(options, args);
107+
108+
if (cmd.hasOption(fileOption)) {
109+
path = new Path(cmd.getOptionValue(fileOption));
110+
} else {
111+
regionName = cmd.getOptionValue(regionOption);
112+
if (StringUtils.isEmpty(regionName)) {
113+
err.println("Region name is not specified.");
114+
formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|"
115+
+ "tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
116+
System.exit(1);
117+
}
118+
columnFamily = cmd.getOptionValue(columnFamilyOption);
119+
if (StringUtils.isEmpty(columnFamily)) {
120+
err.println("Column family is not specified.");
121+
formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|"
122+
+ "tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
123+
System.exit(1);
124+
}
125+
String tableNameWtihNS = cmd.getOptionValue(tableNameOption);
126+
if (StringUtils.isEmpty(tableNameWtihNS)) {
127+
err.println("Table name is not specified.");
128+
formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|"
129+
+ "tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
130+
System.exit(1);
131+
}
132+
TableName tn = TableName.valueOf(tableNameWtihNS);
133+
namespace = tn.getNamespaceAsString();
134+
tableName = tn.getNameAsString();
135+
}
136+
return true;
137+
}
138+
139+
public int run(String[] args) {
140+
if (getConf() == null) {
141+
throw new RuntimeException("A Configuration instance must be provided.");
142+
}
143+
boolean pass = true;
144+
try {
145+
CommonFSUtils.setFsDefault(getConf(), CommonFSUtils.getRootDir(getConf()));
146+
if (!parseOptions(args)) {
147+
return 1;
148+
}
149+
} catch (IOException ex) {
150+
LOG.error("Error parsing command-line options", ex);
151+
return 1;
152+
} catch (ParseException ex) {
153+
LOG.error("Error parsing command-line options", ex);
154+
return 1;
155+
}
156+
FileSystem fs = null;
157+
if (path != null) {
158+
try {
159+
fs = path.getFileSystem(getConf());
160+
if (fs.isDirectory(path)) {
161+
err.println("ERROR, wrong path given: " + path);
162+
return 2;
163+
}
164+
return print(fs, path);
165+
} catch (IOException e) {
166+
LOG.error("Error reading " + path, e);
167+
return 2;
168+
}
169+
} else {
170+
try {
171+
Path root = CommonFSUtils.getRootDir(getConf());
172+
Path baseDir = new Path(root, HConstants.BASE_NAMESPACE_DIR);
173+
Path nameSpacePath = new Path(baseDir, namespace);
174+
Path tablePath = new Path(nameSpacePath, tableName);
175+
Path regionPath = new Path(tablePath, regionName);
176+
Path cfPath = new Path(regionPath, columnFamily);
177+
Path sftPath = new Path(cfPath, StoreFileListFile.TRACK_FILE_DIR);
178+
179+
fs = FileSystem.newInstance(regionPath.toUri(), getConf());
180+
181+
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(sftPath, false);
182+
183+
while (iterator.hasNext()) {
184+
LocatedFileStatus lfs = iterator.next();
185+
if (
186+
lfs.isFile()
187+
&& StoreFileListFile.TRACK_FILE_PATTERN.matcher(lfs.getPath().getName()).matches()
188+
) {
189+
out.println("Printing contents for file " + lfs.getPath().toString());
190+
int ret = print(fs, lfs.getPath());
191+
if (ret != 0) {
192+
pass = false;
193+
}
194+
}
195+
}
196+
} catch (IOException e) {
197+
LOG.error("Error processing " + e);
198+
return 2;
199+
}
200+
}
201+
return pass ? 0 : 2;
202+
}
203+
204+
private int print(FileSystem fs, Path path) throws IOException {
205+
try {
206+
if (!fs.exists(path)) {
207+
err.println("ERROR, file doesnt exist: " + path);
208+
return 2;
209+
}
210+
} catch (IOException e) {
211+
err.println("ERROR, reading file: " + path + e);
212+
return 2;
213+
}
214+
StoreFileList storeFile = StoreFileListFile.load(fs, path);
215+
int end = storeFile.getStoreFileCount();
216+
for (int i = 0; i < end; i++) {
217+
out.println(storeFile.getStoreFile(i).getName());
218+
}
219+
return 0;
220+
}
221+
222+
public static void main(String[] args) throws Exception {
223+
Configuration conf = HBaseConfiguration.create();
224+
int ret = ToolRunner.run(conf, new StoreFileListFilePrettyPrinter(), args);
225+
System.exit(ret);
226+
}
227+
}

0 commit comments

Comments
 (0)