Skip to content

Commit d5e97fe

Browse files
GauthamBanasandraGauthamBanasandra
authored
HDFS-16473. Make HDFS stat tool cross platform (#4145)
* The source files for hdfs_stat uses getopt for parsing the command line arguments. * getopt is available only on Linux and thus, isn't cross platform. * We need to replace getopt with boost::program_options to make this tool cross platform.
1 parent b69ede7 commit d5e97fe

File tree

10 files changed

+423
-89
lines changed

10 files changed

+423
-89
lines changed

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ add_executable(hdfs_tool_tests
3939
hdfs-find-mock.cc
4040
hdfs-ls-mock.cc
4141
hdfs-setrep-mock.cc
42+
hdfs-stat-mock.cc
4243
main.cc)
4344
target_include_directories(hdfs_tool_tests PRIVATE
4445
../tools
@@ -62,6 +63,7 @@ target_include_directories(hdfs_tool_tests PRIVATE
6263
../../tools/hdfs-find
6364
../../tools/hdfs-ls
6465
../../tools/hdfs-setrep
66+
../../tools/hdfs-stat
6567
../../tools/hdfs-cat)
6668
target_link_libraries(hdfs_tool_tests PRIVATE
6769
gmock_main
@@ -84,5 +86,6 @@ target_link_libraries(hdfs_tool_tests PRIVATE
8486
hdfs_find_lib
8587
hdfs_ls_lib
8688
hdfs_setrep_lib
89+
hdfs_stat_lib
8790
hdfs_cat_lib)
8891
add_test(hdfs_tool_tests hdfs_tool_tests)

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#include <functional>
20+
#include <memory>
21+
#include <string>
22+
#include <vector>
23+
24+
#include <gmock/gmock.h>
25+
#include <gtest/gtest.h>
26+
27+
#include "hdfs-stat-mock.h"
28+
#include "hdfs-tool-tests.h"
29+
30+
namespace hdfs::tools::test {
31+
StatMock::~StatMock() = default;
32+
33+
void StatMock::SetExpectations(
34+
std::function<std::unique_ptr<StatMock>()> test_case,
35+
const std::vector<std::string> &args) const {
36+
// Get the pointer to the function that defines the test case
37+
const auto test_case_func =
38+
test_case.target<std::unique_ptr<StatMock> (*)()>();
39+
ASSERT_NE(test_case_func, nullptr);
40+
41+
// Set the expected method calls and their corresponding arguments for each
42+
// test case
43+
if (*test_case_func == &CallHelp<StatMock>) {
44+
EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
45+
return;
46+
}
47+
48+
if (*test_case_func == &PassAPath<StatMock>) {
49+
const auto path = args[0];
50+
EXPECT_CALL(*this, HandlePath(path))
51+
.Times(1)
52+
.WillOnce(testing::Return(true));
53+
}
54+
}
55+
} // namespace hdfs::tools::test

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#ifndef LIBHDFSPP_TOOLS_HDFS_STAT_MOCK
20+
#define LIBHDFSPP_TOOLS_HDFS_STAT_MOCK
21+
22+
#include <functional>
23+
#include <memory>
24+
#include <string>
25+
#include <vector>
26+
27+
#include <gmock/gmock.h>
28+
29+
#include "hdfs-stat.h"
30+
31+
namespace hdfs::tools::test {
32+
/**
33+
* {@class StatMock} is an {@class Stat} whereby it mocks the
34+
* HandleHelp and HandlePath methods for testing their functionality.
35+
*/
36+
class StatMock : public hdfs::tools::Stat {
37+
public:
38+
/**
39+
* {@inheritdoc}
40+
*/
41+
StatMock(const int argc, char **argv) : Stat(argc, argv) {}
42+
43+
// Abiding to the Rule of 5
44+
StatMock(const StatMock &) = delete;
45+
StatMock(StatMock &&) = delete;
46+
StatMock &operator=(const StatMock &) = delete;
47+
StatMock &operator=(StatMock &&) = delete;
48+
~StatMock() override;
49+
50+
/**
51+
* Defines the methods and the corresponding arguments that are expected
52+
* to be called on this instance of {@link HdfsTool} for the given test case.
53+
*
54+
* @param test_case An {@link std::function} object that points to the
55+
* function defining the test case
56+
* @param args The arguments that are passed to this test case
57+
*/
58+
void SetExpectations(std::function<std::unique_ptr<StatMock>()> test_case,
59+
const std::vector<std::string> &args = {}) const;
60+
61+
MOCK_METHOD(bool, HandleHelp, (), (const, override));
62+
63+
MOCK_METHOD(bool, HandlePath, (const std::string &), (const, override));
64+
};
65+
} // namespace hdfs::tools::test
66+
67+
#endif

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include "hdfs-rename-snapshot-mock.h"
4040
#include "hdfs-rm-mock.h"
4141
#include "hdfs-setrep-mock.h"
42+
#include "hdfs-stat-mock.h"
4243
#include "hdfs-tool-test-fixtures.h"
4344
#include "hdfs-tool-tests.h"
4445

@@ -162,6 +163,11 @@ INSTANTIATE_TEST_SUITE_P(
162163
testing::Values(CallHelp<hdfs::tools::test::SetrepMock>,
163164
PassPermissionsAndAPath<hdfs::tools::test::SetrepMock>));
164165

166+
INSTANTIATE_TEST_SUITE_P(
167+
HdfsStat, HdfsToolBasicTest,
168+
testing::Values(CallHelp<hdfs::tools::test::StatMock>,
169+
PassAPath<hdfs::tools::test::StatMock>));
170+
165171
// Negative tests
166172
INSTANTIATE_TEST_SUITE_P(
167173
HdfsAllowSnapshot, HdfsToolNegativeTestThrows,
@@ -265,6 +271,17 @@ INSTANTIATE_TEST_SUITE_P(
265271
PassMOpt<hdfs::tools::test::SetrepMock>,
266272
PassNOpt<hdfs::tools::test::SetrepMock>));
267273

274+
INSTANTIATE_TEST_SUITE_P(
275+
HdfsStat, HdfsToolNegativeTestThrows,
276+
testing::Values(Pass2Paths<hdfs::tools::test::StatMock>,
277+
Pass3Paths<hdfs::tools::test::StatMock>,
278+
PassRecursiveOwnerAndAPath<hdfs::tools::test::StatMock>,
279+
PassRecursive<hdfs::tools::test::StatMock>,
280+
PassRecursivePath<hdfs::tools::test::StatMock>,
281+
PassMPOptsPermissionsAndAPath<hdfs::tools::test::StatMock>,
282+
PassMOpt<hdfs::tools::test::StatMock>,
283+
PassNOpt<hdfs::tools::test::StatMock>));
284+
268285
INSTANTIATE_TEST_SUITE_P(
269286
HdfsRm, HdfsToolNegativeTestNoThrow,
270287
testing::Values(PassRecursive<hdfs::tools::test::RmMock>));

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ add_subdirectory(hdfs-rm)
4949

5050
add_subdirectory(hdfs-ls)
5151

52-
add_executable(hdfs_stat hdfs_stat.cc)
53-
target_link_libraries(hdfs_stat tools_common hdfspp_static)
52+
add_subdirectory(hdfs-stat)
5453

5554
add_subdirectory(hdfs-count)
5655

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
add_library(hdfs_stat_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-stat.cc)
20+
target_include_directories(hdfs_stat_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
21+
target_link_libraries(hdfs_stat_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
22+
23+
add_executable(hdfs_stat main.cc)
24+
target_include_directories(hdfs_stat PRIVATE ../../tools)
25+
target_link_libraries(hdfs_stat PRIVATE hdfs_stat_lib)
26+
27+
install(TARGETS hdfs_stat RUNTIME DESTINATION bin)

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#include <future>
20+
#include <iostream>
21+
#include <memory>
22+
#include <ostream>
23+
#include <sstream>
24+
#include <string>
25+
26+
#include "hdfs-stat.h"
27+
#include "tools_common.h"
28+
29+
namespace hdfs::tools {
30+
Stat::Stat(const int argc, char **argv) : HdfsTool(argc, argv) {}
31+
32+
bool Stat::Initialize() {
33+
auto add_options = opt_desc_.add_options();
34+
add_options("help,h", "Displays the stat information for the given path. The "
35+
"path can be a file or a directory.");
36+
add_options("path", po::value<std::string>(),
37+
"The path in the filesystem for which to display the "
38+
"stat information.");
39+
40+
// We allow only one positional argument to be passed to this tool. An
41+
// exception is thrown if multiple arguments are passed.
42+
pos_opt_desc_.add("path", 1);
43+
44+
po::store(po::command_line_parser(argc_, argv_)
45+
.options(opt_desc_)
46+
.positional(pos_opt_desc_)
47+
.run(),
48+
opt_val_);
49+
po::notify(opt_val_);
50+
return true;
51+
}
52+
53+
std::string Stat::GetDescription() const {
54+
std::stringstream desc;
55+
desc << "Usage: hdfs_stat PATH" << std::endl
56+
<< std::endl
57+
<< "Displays the stat information for the given path." << std::endl
58+
<< "The path can be a file or a directory." << std::endl
59+
<< "Examples:" << std::endl
60+
<< "hdfs_stat hdfs://localhost.localdomain:8020/dir/file" << std::endl;
61+
return desc.str();
62+
}
63+
64+
bool Stat::Do() {
65+
if (!Initialize()) {
66+
std::cerr << "Unable to initialize HDFS stat tool" << std::endl;
67+
return false;
68+
}
69+
70+
if (!ValidateConstraints()) {
71+
std::cout << GetDescription();
72+
return false;
73+
}
74+
75+
if (opt_val_.count("help") > 0) {
76+
return HandleHelp();
77+
}
78+
79+
if (opt_val_.count("path") > 0) {
80+
const auto path = opt_val_["path"].as<std::string>();
81+
return HandlePath(path);
82+
}
83+
84+
return false;
85+
}
86+
87+
bool Stat::HandleHelp() const {
88+
std::cout << GetDescription();
89+
return true;
90+
}
91+
92+
bool Stat::HandlePath(const std::string &path) const {
93+
// Building a URI object from the given uri_path
94+
auto uri = hdfs::parse_path_or_exit(path);
95+
96+
const auto fs = hdfs::doConnect(uri, false);
97+
if (!fs) {
98+
std::cerr << "Could not connect the file system. " << std::endl;
99+
return false;
100+
}
101+
102+
hdfs::StatInfo stat_info;
103+
const auto status = fs->GetFileInfo(uri.get_path(), stat_info);
104+
if (!status.ok()) {
105+
std::cerr << "Error: " << status.ToString() << std::endl;
106+
return false;
107+
}
108+
std::cout << stat_info.str() << std::endl;
109+
return true;
110+
}
111+
} // namespace hdfs::tools

0 commit comments

Comments
 (0)