Skip to content

Commit de27fa0

Browse files
anoopsjohnsunchao
authored andcommitted
HADOOP-17643 WASB : Make metadata checks case insensitive (#3103)
1 parent c257abc commit de27fa0

File tree

1 file changed

+52
-24
lines changed

1 file changed

+52
-24
lines changed

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.util.Iterator;
4141
import java.util.Locale;
4242
import java.util.Map;
43+
import java.util.Map.Entry;
4344
import java.util.Optional;
4445
import java.util.Set;
4546

@@ -181,6 +182,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
181182
*/
182183
public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode";
183184

185+
/**
186+
* Config to control case sensitive metadata key checks/retrieval. If this
187+
* is false, blob metadata keys will be treated case insensitive.
188+
*/
189+
private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive";
184190
private static final String PERMISSION_METADATA_KEY = "hdi_permission";
185191
private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission";
186192
private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder";
@@ -364,6 +370,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
364370

365371
private String delegationToken;
366372

373+
private boolean metadataKeyCaseSensitive;
374+
367375
/** The error message template when container is not accessible. */
368376
public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
369377
+ "account %s in the configuration, and its container %s is not "
@@ -585,6 +593,12 @@ public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentati
585593
LOG.warn("Unable to initialize HBase root as an atomic rename directory.");
586594
}
587595
LOG.debug("Atomic rename directories: {} ", setToString(atomicRenameDirs));
596+
metadataKeyCaseSensitive = conf
597+
.getBoolean(KEY_BLOB_METADATA_KEY_CASE_SENSITIVE, true);
598+
if (!metadataKeyCaseSensitive) {
599+
LOG.info("{} configured as false. Blob metadata will be treated case insensitive.",
600+
KEY_BLOB_METADATA_KEY_CASE_SENSITIVE);
601+
}
588602
}
589603

590604
/**
@@ -1633,15 +1647,24 @@ private static void storeMetadataAttribute(CloudBlobWrapper blob,
16331647
blob.setMetadata(metadata);
16341648
}
16351649

1636-
private static String getMetadataAttribute(CloudBlobWrapper blob,
1650+
private String getMetadataAttribute(HashMap<String, String> metadata,
16371651
String... keyAlternatives) {
1638-
HashMap<String, String> metadata = blob.getMetadata();
16391652
if (null == metadata) {
16401653
return null;
16411654
}
16421655
for (String key : keyAlternatives) {
1643-
if (metadata.containsKey(key)) {
1644-
return metadata.get(key);
1656+
if (metadataKeyCaseSensitive) {
1657+
if (metadata.containsKey(key)) {
1658+
return metadata.get(key);
1659+
}
1660+
} else {
1661+
// See HADOOP-17643 for details on why this case insensitive metadata
1662+
// checks been added
1663+
for (Entry<String, String> entry : metadata.entrySet()) {
1664+
if (key.equalsIgnoreCase(entry.getKey())) {
1665+
return entry.getValue();
1666+
}
1667+
}
16451668
}
16461669
}
16471670
return null;
@@ -1665,7 +1688,7 @@ private static void storePermissionStatus(CloudBlobWrapper blob,
16651688
}
16661689

16671690
private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) {
1668-
String permissionMetadataValue = getMetadataAttribute(blob,
1691+
String permissionMetadataValue = getMetadataAttribute(blob.getMetadata(),
16691692
PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY);
16701693
if (permissionMetadataValue != null) {
16711694
return PermissionStatusJsonSerializer.fromJSONString(
@@ -1713,19 +1736,32 @@ private static void storeLinkAttribute(CloudBlobWrapper blob,
17131736
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
17141737
}
17151738

1716-
private static String getLinkAttributeValue(CloudBlobWrapper blob)
1739+
private String getLinkAttributeValue(CloudBlobWrapper blob)
17171740
throws UnsupportedEncodingException {
1718-
String encodedLinkTarget = getMetadataAttribute(blob,
1741+
String encodedLinkTarget = getMetadataAttribute(blob.getMetadata(),
17191742
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
17201743
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
17211744
return decodeMetadataAttribute(encodedLinkTarget);
17221745
}
17231746

1724-
private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
1747+
private boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
17251748
HashMap<String, String> metadata = blob.getMetadata();
1726-
return null != metadata
1727-
&& (metadata.containsKey(IS_FOLDER_METADATA_KEY) || metadata
1728-
.containsKey(OLD_IS_FOLDER_METADATA_KEY));
1749+
if (null != metadata) {
1750+
if (metadataKeyCaseSensitive) {
1751+
return metadata.containsKey(IS_FOLDER_METADATA_KEY)
1752+
|| metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY);
1753+
} else {
1754+
// See HADOOP-17643 for details on why this case insensitive metadata
1755+
// checks been added
1756+
for (String key : metadata.keySet()) {
1757+
if (key.equalsIgnoreCase(IS_FOLDER_METADATA_KEY)
1758+
|| key.equalsIgnoreCase(OLD_IS_FOLDER_METADATA_KEY)) {
1759+
return true;
1760+
}
1761+
}
1762+
}
1763+
}
1764+
return false;
17291765
}
17301766

17311767
private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
@@ -1740,18 +1776,9 @@ private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
17401776
container.setMetadata(metadata);
17411777
}
17421778

1743-
private static String retrieveVersionAttribute(
1744-
CloudBlobContainerWrapper container) {
1745-
HashMap<String, String> metadata = container.getMetadata();
1746-
if (metadata == null) {
1747-
return null;
1748-
} else if (metadata.containsKey(VERSION_METADATA_KEY)) {
1749-
return metadata.get(VERSION_METADATA_KEY);
1750-
} else if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) {
1751-
return metadata.get(OLD_VERSION_METADATA_KEY);
1752-
} else {
1753-
return null;
1754-
}
1779+
private String retrieveVersionAttribute(CloudBlobContainerWrapper container) {
1780+
return getMetadataAttribute(container.getMetadata(), VERSION_METADATA_KEY,
1781+
OLD_VERSION_METADATA_KEY);
17551782
}
17561783

17571784
@Override
@@ -2246,7 +2273,8 @@ public byte[] retrieveAttribute(String key, String attribute) throws IOException
22462273
CloudBlobWrapper blob = getBlobReference(key);
22472274
blob.downloadAttributes(getInstrumentedContext());
22482275

2249-
String value = getMetadataAttribute(blob, ensureValidAttributeName(attribute));
2276+
String value = getMetadataAttribute(blob.getMetadata(),
2277+
ensureValidAttributeName(attribute));
22502278
value = decodeMetadataAttribute(value);
22512279
return value == null ? null : value.getBytes(METADATA_ENCODING);
22522280
} catch (Exception e) {

0 commit comments

Comments
 (0)