40
40
import java .util .Iterator ;
41
41
import java .util .Locale ;
42
42
import java .util .Map ;
43
+ import java .util .Map .Entry ;
43
44
import java .util .Optional ;
44
45
import java .util .Set ;
45
46
@@ -181,6 +182,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
181
182
*/
182
183
public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode" ;
183
184
185
+ /**
186
+ * Config to control case sensitive metadata key checks/retrieval. If this
187
+ * is false, blob metadata keys will be treated case insensitive.
188
+ */
189
+ private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive" ;
184
190
private static final String PERMISSION_METADATA_KEY = "hdi_permission" ;
185
191
private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission" ;
186
192
private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder" ;
@@ -364,6 +370,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
364
370
365
371
private String delegationToken ;
366
372
373
+ private boolean metadataKeyCaseSensitive ;
374
+
367
375
/** The error message template when container is not accessible. */
368
376
public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
369
377
+ "account %s in the configuration, and its container %s is not "
@@ -585,6 +593,12 @@ public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentati
585
593
LOG .warn ("Unable to initialize HBase root as an atomic rename directory." );
586
594
}
587
595
LOG .debug ("Atomic rename directories: {} " , setToString (atomicRenameDirs ));
596
+ metadataKeyCaseSensitive = conf
597
+ .getBoolean (KEY_BLOB_METADATA_KEY_CASE_SENSITIVE , true );
598
+ if (!metadataKeyCaseSensitive ) {
599
+ LOG .info ("{} configured as false. Blob metadata will be treated case insensitive." ,
600
+ KEY_BLOB_METADATA_KEY_CASE_SENSITIVE );
601
+ }
588
602
}
589
603
590
604
/**
@@ -1633,15 +1647,24 @@ private static void storeMetadataAttribute(CloudBlobWrapper blob,
1633
1647
blob .setMetadata (metadata );
1634
1648
}
1635
1649
1636
- private static String getMetadataAttribute (CloudBlobWrapper blob ,
1650
+ private String getMetadataAttribute (HashMap < String , String > metadata ,
1637
1651
String ... keyAlternatives ) {
1638
- HashMap <String , String > metadata = blob .getMetadata ();
1639
1652
if (null == metadata ) {
1640
1653
return null ;
1641
1654
}
1642
1655
for (String key : keyAlternatives ) {
1643
- if (metadata .containsKey (key )) {
1644
- return metadata .get (key );
1656
+ if (metadataKeyCaseSensitive ) {
1657
+ if (metadata .containsKey (key )) {
1658
+ return metadata .get (key );
1659
+ }
1660
+ } else {
1661
+ // See HADOOP-17643 for details on why this case insensitive metadata
1662
+ // checks been added
1663
+ for (Entry <String , String > entry : metadata .entrySet ()) {
1664
+ if (key .equalsIgnoreCase (entry .getKey ())) {
1665
+ return entry .getValue ();
1666
+ }
1667
+ }
1645
1668
}
1646
1669
}
1647
1670
return null ;
@@ -1665,7 +1688,7 @@ private static void storePermissionStatus(CloudBlobWrapper blob,
1665
1688
}
1666
1689
1667
1690
private PermissionStatus getPermissionStatus (CloudBlobWrapper blob ) {
1668
- String permissionMetadataValue = getMetadataAttribute (blob ,
1691
+ String permissionMetadataValue = getMetadataAttribute (blob . getMetadata () ,
1669
1692
PERMISSION_METADATA_KEY , OLD_PERMISSION_METADATA_KEY );
1670
1693
if (permissionMetadataValue != null ) {
1671
1694
return PermissionStatusJsonSerializer .fromJSONString (
@@ -1713,19 +1736,32 @@ private static void storeLinkAttribute(CloudBlobWrapper blob,
1713
1736
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY );
1714
1737
}
1715
1738
1716
- private static String getLinkAttributeValue (CloudBlobWrapper blob )
1739
+ private String getLinkAttributeValue (CloudBlobWrapper blob )
1717
1740
throws UnsupportedEncodingException {
1718
- String encodedLinkTarget = getMetadataAttribute (blob ,
1741
+ String encodedLinkTarget = getMetadataAttribute (blob . getMetadata () ,
1719
1742
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY ,
1720
1743
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY );
1721
1744
return decodeMetadataAttribute (encodedLinkTarget );
1722
1745
}
1723
1746
1724
- private static boolean retrieveFolderAttribute (CloudBlobWrapper blob ) {
1747
+ private boolean retrieveFolderAttribute (CloudBlobWrapper blob ) {
1725
1748
HashMap <String , String > metadata = blob .getMetadata ();
1726
- return null != metadata
1727
- && (metadata .containsKey (IS_FOLDER_METADATA_KEY ) || metadata
1728
- .containsKey (OLD_IS_FOLDER_METADATA_KEY ));
1749
+ if (null != metadata ) {
1750
+ if (metadataKeyCaseSensitive ) {
1751
+ return metadata .containsKey (IS_FOLDER_METADATA_KEY )
1752
+ || metadata .containsKey (OLD_IS_FOLDER_METADATA_KEY );
1753
+ } else {
1754
+ // See HADOOP-17643 for details on why this case insensitive metadata
1755
+ // checks been added
1756
+ for (String key : metadata .keySet ()) {
1757
+ if (key .equalsIgnoreCase (IS_FOLDER_METADATA_KEY )
1758
+ || key .equalsIgnoreCase (OLD_IS_FOLDER_METADATA_KEY )) {
1759
+ return true ;
1760
+ }
1761
+ }
1762
+ }
1763
+ }
1764
+ return false ;
1729
1765
}
1730
1766
1731
1767
private static void storeVersionAttribute (CloudBlobContainerWrapper container ) {
@@ -1740,18 +1776,9 @@ private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
1740
1776
container .setMetadata (metadata );
1741
1777
}
1742
1778
1743
- private static String retrieveVersionAttribute (
1744
- CloudBlobContainerWrapper container ) {
1745
- HashMap <String , String > metadata = container .getMetadata ();
1746
- if (metadata == null ) {
1747
- return null ;
1748
- } else if (metadata .containsKey (VERSION_METADATA_KEY )) {
1749
- return metadata .get (VERSION_METADATA_KEY );
1750
- } else if (metadata .containsKey (OLD_VERSION_METADATA_KEY )) {
1751
- return metadata .get (OLD_VERSION_METADATA_KEY );
1752
- } else {
1753
- return null ;
1754
- }
1779
+ private String retrieveVersionAttribute (CloudBlobContainerWrapper container ) {
1780
+ return getMetadataAttribute (container .getMetadata (), VERSION_METADATA_KEY ,
1781
+ OLD_VERSION_METADATA_KEY );
1755
1782
}
1756
1783
1757
1784
@ Override
@@ -2246,7 +2273,8 @@ public byte[] retrieveAttribute(String key, String attribute) throws IOException
2246
2273
CloudBlobWrapper blob = getBlobReference (key );
2247
2274
blob .downloadAttributes (getInstrumentedContext ());
2248
2275
2249
- String value = getMetadataAttribute (blob , ensureValidAttributeName (attribute ));
2276
+ String value = getMetadataAttribute (blob .getMetadata (),
2277
+ ensureValidAttributeName (attribute ));
2250
2278
value = decodeMetadataAttribute (value );
2251
2279
return value == null ? null : value .getBytes (METADATA_ENCODING );
2252
2280
} catch (Exception e ) {
0 commit comments