Workaround for nvidia bug, nicked from hashcat.

openwall · Feb 1, 2018 · f99d183 · f99d183
1 parent 608b5d1
commit f99d183
Show file tree

Hide file tree

Showing 16 changed files with 83 additions and 63 deletions.
diff --git a/src/opencl/bitlocker_kernel.cl b/src/opencl/bitlocker_kernel.cl
@@ -18,7 +18,7 @@
 #include "opencl_misc.h"
 #include "opencl_bitlocker.h"
 
-__constant unsigned int TS0[256] = {
+__const_a8 unsigned int TS0[256] = {
 	0xC66363A5U, 0xF87C7C84U, 0xEE777799U, 0xF67B7B8DU, 0xFFF2F20DU,
 	0xD66B6BBDU, 0xDE6F6FB1U, 0x91C5C554U,
 	0x60303050U, 0x02010103U, 0xCE6767A9U, 0x562B2B7DU, 0xE7FEFE19U,
@@ -85,7 +85,7 @@ __constant unsigned int TS0[256] = {
 	0xA85454FCU, 0x6DBBBBD6U, 0x2C16163AU
 };
 
-__constant unsigned int TS1[256] = {
+__const_a8 unsigned int TS1[256] = {
 	0xA5C66363U, 0x84F87C7CU, 0x99EE7777U, 0x8DF67B7BU, 0x0DFFF2F2U,
 	0xBDD66B6BU, 0xB1DE6F6FU, 0x5491C5C5U,
 	0x50603030U, 0x03020101U, 0xA9CE6767U, 0x7D562B2BU, 0x19E7FEFEU,
@@ -152,7 +152,7 @@ __constant unsigned int TS1[256] = {
 	0xFCA85454U, 0xD66DBBBBU, 0x3A2C1616U
 };
 
-__constant unsigned int TS2[256] = {
+__const_a8 unsigned int TS2[256] = {
 	0x63A5C663U, 0x7C84F87CU, 0x7799EE77U, 0x7B8DF67BU, 0xF20DFFF2U,
 	0x6BBDD66BU, 0x6FB1DE6FU, 0xC55491C5U,
 	0x30506030U, 0x01030201U, 0x67A9CE67U, 0x2B7D562BU, 0xFE19E7FEU,
@@ -219,7 +219,7 @@ __constant unsigned int TS2[256] = {
 	0x54FCA854U, 0xBBD66DBBU, 0x163A2C16U
 };
 
-__constant unsigned int TS3[256] = {
+__const_a8 unsigned int TS3[256] = {
 	0x6363A5C6U, 0x7C7C84F8U, 0x777799EEU, 0x7B7B8DF6U, 0xF2F20DFFU,
 	0x6B6BBDD6U, 0x6F6FB1DEU, 0xC5C55491U,
 	0x30305060U, 0x01010302U, 0x6767A9CEU, 0x2B2B7D56U, 0xFEFE19E7U,

diff --git a/src/opencl/cryptmd5_kernel.cl b/src/opencl/cryptmd5_kernel.cl
@@ -127,7 +127,7 @@ typedef struct {
 
 __constant uchar cl_md5_salt_prefix[] = "$1$";
 __constant uchar cl_apr1_salt_prefix[] = "$apr1$";
-__constant uchar g[] =
+__const_a8 uchar g[] =
     { 0, 7, 3, 5, 3, 7, 1, 6, 3, 5, 3, 7, 1, 7, 2, 5, 3, 7, 1, 7, 3, 4, 3, 7,
 	    1, 7, 3, 5, 2, 7, 1, 7, 3, 5, 3, 6, 1, 7, 3, 5, 3, 7 };
 

diff --git a/src/opencl/lotus5_kernel.cl b/src/opencl/lotus5_kernel.cl
@@ -16,7 +16,7 @@
 #define MAYBE_CONSTANT __local const
 #endif
 
-static __constant uint magic_table[256] = {
+__const_a8 uint magic_table[256] = {
   0xbd, 0x56, 0xea, 0xf2, 0xa2, 0xf1, 0xac, 0x2a,
   0xb0, 0x93, 0xd1, 0x9c, 0x1b, 0x33, 0xfd, 0xd0,
   0x30, 0x04, 0xb6, 0xdc, 0x7d, 0xdf, 0x32, 0x4b,

diff --git a/src/opencl/office_kernel.cl b/src/opencl/office_kernel.cl
@@ -224,8 +224,8 @@ inline void Decrypt(__constant ms_office_salt *salt,
 	AES_cbc_decrypt(encryptedVerifier, decryptedVerifier, length, &akey, iv);
 }
 
-__constant uint InputBlockKeyInt[] = { 0xfea7d276, 0x3b4b9e79 };
-__constant uint ValueBlockKeyInt[] = { 0xd7aa0f6d, 0x3061344e };
+__const_a8 uint InputBlockKeyInt[] = { 0xfea7d276, 0x3b4b9e79 };
+__const_a8 uint ValueBlockKeyInt[] = { 0xd7aa0f6d, 0x3061344e };
 
 __kernel
 void Generate2010key(__global ms_office_state *state,

diff --git a/src/opencl/wpapsk_kernel.cl b/src/opencl/wpapsk_kernel.cl
@@ -188,7 +188,7 @@ void wpapsk_pass2(MAYBE_CONSTANT wpapsk_salt *salt,
 
 //__constant uchar *text = "Pairwise key expansion\0";
 //__constant uint text[6] = { 0x72696150, 0x65736977, 0x79656b20, 0x70786520, 0x69736e61, 0x00006e6f };
-__constant uint text[6] = { 0x50616972, 0x77697365, 0x206b6579, 0x20657870, 0x616e7369, 0x6f6e0000 };
+__const_a8 uint text[6] = { 0x50616972, 0x77697365, 0x206b6579, 0x20657870, 0x616e7369, 0x6f6e0000 };
 
 inline void prf_512(const MAYBE_VECTOR_UINT *key,
                     MAYBE_CONSTANT uint *data,

diff --git a/src/opencl_aes.h b/src/opencl_aes.h
@@ -63,7 +63,7 @@ typedef struct aes_key_st {
  */
 #undef FULL_UNROLL
 
-static __constant uint Te0[256] = {
+__const_a8 uint Te0[256] = {
 	0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
 	0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
 	0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
@@ -129,7 +129,8 @@ static __constant uint Te0[256] = {
 	0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
 	0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
 };
-static __constant uint Te1[256] = {
+
+__const_a8 uint Te1[256] = {
 	0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
 	0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
 	0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
@@ -195,7 +196,8 @@ static __constant uint Te1[256] = {
 	0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
 	0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
 };
-static __constant uint Te2[256] = {
+
+__const_a8 uint Te2[256] = {
 	0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
 	0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
 	0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
@@ -261,7 +263,8 @@ static __constant uint Te2[256] = {
 	0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
 	0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
 };
-static __constant uint Te3[256] = {
+
+__const_a8 uint Te3[256] = {
 	0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
 	0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
 	0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
@@ -328,7 +331,7 @@ static __constant uint Te3[256] = {
 	0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
 };
 
-static __constant uint Td0[256] = {
+__const_a8 uint Td0[256] = {
 	0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
 	0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
 	0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
@@ -394,7 +397,8 @@ static __constant uint Td0[256] = {
 	0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
 	0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
 };
-static __constant uint Td1[256] = {
+
+__const_a8 uint Td1[256] = {
 	0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
 	0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
 	0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
@@ -460,7 +464,8 @@ static __constant uint Td1[256] = {
 	0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
 	0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
 };
-static __constant uint Td2[256] = {
+
+__const_a8 uint Td2[256] = {
 	0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
 	0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
 	0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
@@ -526,7 +531,8 @@ static __constant uint Td2[256] = {
 	0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
 	0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
 };
-static __constant uint Td3[256] = {
+
+__const_a8 uint Td3[256] = {
 	0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
 	0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
 	0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
@@ -592,7 +598,8 @@ static __constant uint Td3[256] = {
 	0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
 	0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
 };
-static __constant uchar Td4[256] = {
+
+__const_a8 uchar Td4[256] = {
 	0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
 	0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
 	0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
@@ -626,7 +633,8 @@ static __constant uchar Td4[256] = {
 	0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
 	0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
 };
-static __constant uint rcon[] = {
+
+__const_a8 uint rcon[] = {
 	0x01000000, 0x02000000, 0x04000000, 0x08000000,
 	0x10000000, 0x20000000, 0x40000000, 0x80000000,
 	0x1B000000, 0x36000000,

diff --git a/src/opencl_chacha.h b/src/opencl_chacha.h
@@ -55,8 +55,8 @@ typedef struct chacha_ctx_s {
   a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \
   c = PLUS(c,d); b = ROTATE(XOR(b,c), 7);
 
-__constant char sigma[16] = "expand 32-byte k";
-__constant char tau[16] = "expand 16-byte k";
+__const_a8 char sigma[16] = "expand 32-byte k";
+__const_a8 char tau[16] = "expand 16-byte k";
 
 inline
 void chacha_keysetup(chacha_ctx *x, const uchar *k, uint kbits)

diff --git a/src/opencl_keccak.h b/src/opencl_keccak.h
@@ -7,26 +7,31 @@
  * but not liability.
  */
 
+#ifndef _OPENCL_KECCAK_H
+#define _OPENCL_KECCAK_H
+
+#include "opencl_misc.h"
+
 /******** The Keccak-f[1600] permutation ********/
 
 /*** Constants. ***/
-__constant uint rho[24] =
+__const_a8 uint rho[24] =
 {
      1,  3,  6, 10, 15, 21,
     28, 36, 45, 55,  2, 14,
     27, 41, 56,  8, 25, 43,
     62, 18, 39, 61, 20, 44
 };
 
-__constant uint pi[24] =
+__const_a8 uint pi[24] =
 {
     10,  7, 11, 17, 18, 3,
      5, 16,  8, 21, 24, 4,
     15, 23, 19, 13, 12, 2,
     20, 14, 22,  9,  6, 1
 };
 
-__constant uint64_t RC[24] =
+__const_a8 uint64_t RC[24] =
 {
   0x0000000000000001UL, 0x0000000000008082UL, 0x800000000000808aUL,
   0x8000000080008000UL, 0x000000000000808bUL, 0x0000000080000001UL,
@@ -169,3 +174,5 @@ inline void hash(uint8_t* out, uint outlen, const uint8_t* in, uint inlen,
   }
 
 defkeccak(256)
+
+#endif /* _OPENCL_KECCAK_H */
diff --git a/src/opencl_misc.h b/src/opencl_misc.h
@@ -28,6 +28,9 @@ typedef int int32_t;
 typedef ulong uint64_t;
 typedef long int64_t;
 
+/* Nvidia bug workaround nicked from hashcat. These are for __constant arrays */
+#define __const_a8	__constant __attribute__ ((aligned (8)))
+
 #if SIZEOF_SIZE_T == 8
 typedef uint64_t host_size_t;
 #else

diff --git a/src/opencl_rc4.h b/src/opencl_rc4.h
@@ -29,7 +29,7 @@
 #endif
 
 #ifdef RC4_IV32
-__constant uint rc4_iv[64] = { 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
+__const_a8 uint rc4_iv[64] = { 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
                                0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
                                0x23222120, 0x27262524, 0x2b2a2928, 0x2f2e2d2c,
                                0x33323130, 0x37363534, 0x3b3a3938, 0x3f3e3d3c,

diff --git a/src/opencl_sha2.h b/src/opencl_sha2.h
@@ -32,12 +32,12 @@
 
 #define ror(x, n)	rotate(x, 32U-(n))
 
-__constant uint h[] = {
+__const_a8 uint h[] = {
 	0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
 	0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
 };
 
-__constant uint k[] = {
+__const_a8 uint k[] = {
 	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
 	0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
 	0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
@@ -327,7 +327,7 @@ __constant uint k[] = {
 #define Maj(x, y, z) ((x & y) | (z & (x | y)))
 #endif
 
-__constant ulong K[] = {
+__const_a8 ulong K[] = {
 	0x428a2f98d728ae22UL, 0x7137449123ef65cdUL, 0xb5c0fbcfec4d3b2fUL,
 	0xe9b5dba58189dbbcUL, 0x3956c25bf348b538UL, 0x59f111f1b605d019UL,
 	0x923f82a4af194f9bUL, 0xab1c5ed5da6d8118UL, 0xd807aa98a3030242UL,

diff --git a/src/opencl_sha256.h b/src/opencl_sha256.h
@@ -44,7 +44,7 @@
 #define H7      0x5be0cd19U
 
 #ifdef _OPENCL_COMPILER
-__constant uint32_t k[] = {
+__const_a8 uint32_t k[] = {
 	0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U,
 	0x3956c25bU, 0x59f111f1U, 0x923f82a4U, 0xab1c5ed5U,
 	0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U,
@@ -63,13 +63,13 @@ __constant uint32_t k[] = {
 	0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, 0xc67178f2U
 };
 
-__constant uint32_t clear_mask[] = {
+__const_a8 uint32_t clear_mask[] = {
 	0xffffffffU, 0x000000ffU,   //0,   8bits
 	0x0000ffffU, 0x00ffffffU,   //16, 24bits
 	0xffffffffU                 //32    bits
 };
 
-__constant uint32_t clear_be_mask[] = {
+__const_a8 uint32_t clear_be_mask[] = {
 	0xffffffffU, 0xff000000U,   //0,   8bits
 	0xffff0000U, 0xffffff00U,   //16, 24bits
 	0xffffffffU                 //32    bits

diff --git a/src/opencl_sha2_common.h b/src/opencl_sha2_common.h
@@ -26,7 +26,7 @@
 //1: 3      7,35                    =>  2
 //1: 7      3,9,15,27,33,39             =>  6
 //1: 3,7    1,5,11,13,17,19,23,25,29,31,37,41   => 12
-__constant int loop_index[] = {
+__const_a8 int loop_index[] = {
 	0, /* 0,000 */ 7, /* 1,111 */ 3, /* 2,011 */ 5, /* 3,101 */
 	3, /* 4,011 */ 7, /* 5,111 */ 1, /* 6,001 */ 6, /* 7,110 */
 	3, /* 8,011 */ 5, /* 9,101 */ 3, /*10,011 */ 7, /*11,111 */
@@ -40,7 +40,7 @@ __constant int loop_index[] = {
 	3, /*40,011 */ 7,           /*41,111 */
 };
 
-__constant int generator_index[] = {
+__const_a8 int generator_index[] = {
 	0,                          /*  0, 000 */
 	6,                          /*  6, 001 */
 	14,                         /* 14, 010 */

diff --git a/src/opencl_sha512.h b/src/opencl_sha512.h
@@ -48,7 +48,7 @@
 #define H7      0x5be0cd19137e2179UL
 
 #ifdef _OPENCL_COMPILER
-__constant uint64_t k[] = {
+__const_a8 uint64_t k[] = {
     0x428a2f98d728ae22UL, 0x7137449123ef65cdUL, 0xb5c0fbcfec4d3b2fUL,
     0xe9b5dba58189dbbcUL, 0x3956c25bf348b538UL, 0x59f111f1b605d019UL,
     0x923f82a4af194f9bUL, 0xab1c5ed5da6d8118UL, 0xd807aa98a3030242UL,
@@ -78,7 +78,7 @@ __constant uint64_t k[] = {
     0x5fcb6fab3ad6faecUL, 0x6c44198c4a475817UL
 };
 
-__constant uint64_t clear_mask[] = {
+__const_a8 uint64_t clear_mask[] = {
 	0xffffffffffffffffUL, 0x00000000000000ffUL, //0,   8bits
 	0x000000000000ffffUL, 0x0000000000ffffffUL, //16, 24bits
 	0x00000000ffffffffUL, 0x000000ffffffffffUL, //32, 40bits

diff --git a/src/opencl_twofish.h b/src/opencl_twofish.h
@@ -103,7 +103,7 @@ typedef Twofish_UInt32 UInt32;
     (p)[3] = (Byte)(((v) >> 24) & 0xff)
 #endif
 
-__constant Byte t_table[2][4][16] =
+__const_a8 Byte t_table[2][4][16] =
 {
 	{
 		{0x8,0x1,0x7,0xD,0x6,0xF,0x3,0x2,0x0,0xB,0x5,0x9,0xE,0xC,0xA,0x4},
@@ -127,7 +127,7 @@ typedef UInt32 Qtype;
 typedef Byte Qtype;
 #endif
 
-__constant Qtype q_table[2][256] =
+__const_a8 Qtype q_table[2][256] =
 {
 	{
 		0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78,
@@ -179,7 +179,7 @@ __constant Qtype q_table[2][256] =
 	}
 };
 
-__constant UInt32 MDS_table[4][256] =
+__const_a8 UInt32 MDS_table[4][256] =
 {
 	{ 0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B,
 	  0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B,
@@ -358,7 +358,7 @@ __constant UInt32 MDS_table[4][256] =
 	  0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8 }
 };
 
-__constant UInt32 mds_poly_divx_const[] = { 0, 0xb4 };
+__const_a8 UInt32 mds_poly_divx_const[] = { 0, 0xb4 };
 
 #define q0  q_table[0]
 #define q1  q_table[1]
@@ -424,8 +424,8 @@ void fill_keyed_sboxes(Byte S[], int kCycles, Twofish_key *xkey)
 	}
 }
 
-__constant uint rs_poly_const[] = { 0, 0x14d };
-__constant uint rs_poly_div_const[] = { 0, 0xa6 };
+__const_a8 uint rs_poly_const[] = { 0, 0x14d };
+__const_a8 uint rs_poly_div_const[] = { 0, 0xa6 };
 
 inline
 void Twofish_prepare_key(__private Byte key[], int key_len, Twofish_key *xkey)