Skip to content

Commit 3ca1529

Browse files
umamaheswararaoumamaheswararao
authored
HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G.
1 parent cc671b1 commit 3ca1529

File tree

3 files changed

+136
-1
lines changed

3 files changed

+136
-1
lines changed

‎hadoop-common-project/hadoop-common/src/main/resources/core-default.xml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,116 @@
952952
</description>
953953
</property>
954954

955+
<property>
956+
<name>fs.viewfs.overload.scheme.target.hdfs.impl</name>
957+
<value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
958+
<description>The DistributedFileSystem for view file system overload scheme
959+
when child file system and ViewFSOverloadScheme's schemes are hdfs.
960+
</description>
961+
</property>
962+
963+
<property>
964+
<name>fs.viewfs.overload.scheme.target.s3a.impl</name>
965+
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
966+
<description>The S3AFileSystem for view file system overload scheme when
967+
child file system and ViewFSOverloadScheme's schemes are s3a.</description>
968+
</property>
969+
970+
<property>
971+
<name>fs.viewfs.overload.scheme.target.o3fs.impl</name>
972+
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
973+
<description>The OzoneFileSystem for view file system overload scheme when
974+
child file system and ViewFSOverloadScheme's schemes are o3fs.</description>
975+
</property>
976+
977+
<property>
978+
<name>fs.viewfs.overload.scheme.target.ftp.impl</name>
979+
<value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
980+
<description>The FTPFileSystem for view file system overload scheme when
981+
child file system and ViewFSOverloadScheme's schemes are ftp.
982+
</description>
983+
</property>
984+
985+
<property>
986+
<name>fs.viewfs.overload.scheme.target.webhdfs.impl</name>
987+
<value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value>
988+
<description>The WebHdfsFileSystem for view file system overload scheme when
989+
child file system and ViewFSOverloadScheme's schemes are webhdfs.
990+
</description>
991+
</property>
992+
993+
<property>
994+
<name>fs.viewfs.overload.scheme.target.swebhdfs.impl</name>
995+
<value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value>
996+
<description>The SWebHdfsFileSystem for view file system overload scheme when
997+
child file system and ViewFSOverloadScheme's schemes are swebhdfs.
998+
</description>
999+
</property>
1000+
1001+
<property>
1002+
<name>fs.viewfs.overload.scheme.target.file.impl</name>
1003+
<value>org.apache.hadoop.fs.LocalFileSystem</value>
1004+
<description>The LocalFileSystem for view file system overload scheme when
1005+
child file system and ViewFSOverloadScheme's schemes are file.
1006+
</description>
1007+
</property>
1008+
1009+
<property>
1010+
<name>fs.viewfs.overload.scheme.target.abfs.impl</name>
1011+
<value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
1012+
<description>The AzureBlobFileSystem for view file system overload scheme
1013+
when child file system and ViewFSOverloadScheme's schemes are abfs.
1014+
</description>
1015+
</property>
1016+
1017+
<property>
1018+
<name>fs.viewfs.overload.scheme.target.abfss.impl</name>
1019+
<value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
1020+
<description>The SecureAzureBlobFileSystem for view file system overload
1021+
scheme when child file system and ViewFSOverloadScheme's schemes are abfss.
1022+
</description>
1023+
</property>
1024+
1025+
<property>
1026+
<name>fs.viewfs.overload.scheme.target.wasb.impl</name>
1027+
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
1028+
<description>The NativeAzureFileSystem for view file system overload scheme
1029+
when child file system and ViewFSOverloadScheme's schemes are wasb.
1030+
</description>
1031+
</property>
1032+
1033+
<property>
1034+
<name>fs.viewfs.overload.scheme.target.swift.impl</name>
1035+
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
1036+
<description>The SwiftNativeFileSystem for view file system overload scheme
1037+
when child file system and ViewFSOverloadScheme's schemes are swift.
1038+
</description>
1039+
</property>
1040+
1041+
<property>
1042+
<name>fs.viewfs.overload.scheme.target.oss.impl</name>
1043+
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
1044+
<description>The AliyunOSSFileSystem for view file system overload scheme
1045+
when child file system and ViewFSOverloadScheme's schemes are oss.
1046+
</description>
1047+
</property>
1048+
1049+
<property>
1050+
<name>fs.viewfs.overload.scheme.target.http.impl</name>
1051+
<value>org.apache.hadoop.fs.http.HttpFileSystem</value>
1052+
<description>The HttpFileSystem for view file system overload scheme
1053+
when child file system and ViewFSOverloadScheme's schemes are http.
1054+
</description>
1055+
</property>
1056+
1057+
<property>
1058+
<name>fs.viewfs.overload.scheme.target.https.impl</name>
1059+
<value>org.apache.hadoop.fs.http.HttpsFileSystem</value>
1060+
<description>The HttpsFileSystem for view file system overload scheme
1061+
when child file system and ViewFSOverloadScheme's schemes are https.
1062+
</description>
1063+
</property>
1064+
9551065
<property>
9561066
<name>fs.AbstractFileSystem.ftp.impl</name>
9571067
<value>org.apache.hadoop.fs.ftp.FtpFs</value>

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,24 @@ public void initializeMemberVariables() {
124124
xmlPrefixToSkipCompare.add("fs.adl.");
125125
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
126126

127+
// ViewfsOverloadScheme target fs impl property keys are dynamically
128+
// constructed and they are advanced props.
129+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfs.impl");
130+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl");
131+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl");
132+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl");
133+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl");
134+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl");
135+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl");
136+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.o3fs.impl");
137+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.oss.impl");
138+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
139+
xmlPropsToSkipCompare.
140+
add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
141+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
142+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
143+
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
144+
127145
// Azure properties are in a different class
128146
// - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
129147
// - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,13 +313,17 @@ public void testCreateOnRootShouldFailEvenFallBackMountLinkConfigured()
313313
* Note: Above links created because to make fs initialization success.
314314
* Otherwise will not proceed if no mount links.
315315
*
316-
* Don't set fs.viewfs.overload.scheme.target.hdfs.impl property.
316+
* Unset fs.viewfs.overload.scheme.target.hdfs.impl property.
317317
* So, OverloadScheme target fs initialization will fail.
318318
*/
319319
@Test(expected = UnsupportedFileSystemException.class, timeout = 30000)
320320
public void testInvalidOverloadSchemeTargetFS() throws Exception {
321321
final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER);
322+
String mountTableIfSet = conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
322323
conf = new Configuration();
324+
if (mountTableIfSet != null) {
325+
conf.set(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH, mountTableIfSet);
326+
}
323327
addMountLinks(defaultFSURI.getAuthority(),
324328
new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER,
325329
Constants.CONFIG_VIEWFS_LINK_FALLBACK },
@@ -331,6 +335,9 @@ public void testInvalidOverloadSchemeTargetFS() throws Exception {
331335
defaultFSURI.toString());
332336
conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME),
333337
ViewFileSystemOverloadScheme.class.getName());
338+
conf.unset(String.format(
339+
FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
340+
HDFS_SCHEME));
334341

335342
try (FileSystem fs = FileSystem.get(conf)) {
336343
fs.createNewFile(new Path("/onRootWhenFallBack"));

0 commit comments

Comments
 (0)