Use processor.cpp for CRC32C dispatch

yuyichao · yuyichao · commit e85e1da31ed4 · 2017-05-30T14:49:46.000-04:00
diff --git a/src/Makefile b/src/Makefile
@@ -45,7 +45,7 @@ SRCS := \
 	simplevector APInt-C runtime_intrinsics runtime_ccall precompile \
 	threadgroup threading stackwalk gc gc-debug gc-pages method \
 	jlapi signal-handling safepoint jloptions timing subtype rtutils \
-	processor
+	processor crc32c
 
 ifeq ($(USEMSVC), 1)
 SRCS += getopt
diff --git a/src/crc32c-tables.c b/src/crc32c-tables.c
@@ -16,7 +16,7 @@ static const uint32_t crc32c_table[8][256] = {
     { 0,1228700967,2457401934,3678701417,555582061,1747058506,3009771555,4200137988,1111164122,185039357,3494117012,2575270835,1663469239,706411408,4049501433,3093430750,2222328244,3444208787,370078714,1597148893,2775288793,3965187838,924021143,2117012656,3326938478,2406576201,1412822816,487164423,3880816387,2926375460,1965585741,1007945834,218129817,1144789182,2675482583,3594838768,740157428,1696701139,3194297786,4149829789,1329291587,101129316,3712195341,2491409962,1848042286,656055817,4234025312,3043124295,2306239533,3226079498,453940835,1379068740,2825645632,3780612967,974328846,1932486953,3410847991,2188449232,1496683193,269086622,3931171482,2741802941,2015891668,823422451,436259634,1396487701,2289578364,3242478683,991775071,1914778744,2842014481,3763981878,1480314856,285717199,3393402278,2206156929,2032553349,807022754,3948853195,2724383468,2658583174,3612000161,202258632,1160922607,3211477227,4132912588,756267685,1680852866,3696084572,2507258747,1312111634,118047029,4249895985,3026991382,1864941183,638894936,385920683,1581044620,2239255781,3427019202,907881670,2132890081,2758137480,3982076847,1429973617,470275926,3343077439,2390699288,1948657692,1025135931,3864973906,2942480245,2474026783,3662338616,17718609,1211244662,2993366386,4216805461,538173244,1764729371,3511526341,2557599458,1127569803,168371372,4031783336,3110886543,1646844902,722773697,872519268,2101209923,2792975402,4014280973,354161673,1545627950,2271538759,3461911392,1983550142,1057419161,3829557488,2910721495,1462178003,505114100,3311397533,2355337146,2960629712,4182500087,571434398,1798510777,2439650749,3629539482,51570675,1244568276,4065106698,3144738349,1614045508,688397411,3545307495,2590860352,1093264169,135634446,956429309,1883082458,2876836275,3796202644,404517264,1361054903,2321845214,3277387513,2067461927,839289344,3913420137,2692640846,1512535370,320538733,3361705732,2170810915,3178756681,4098590574,789512199,1714650400,2624223268,3579184387,236094058,1194262349,4283235987,3060827060,1832125661,604535290,3729882366,2540503513,1277789872,85326743,771841366,1732059249,3162089240,4114995775,253550395,1176543772,2640586101,3562559570,1815763340,621159595,4265780162,3078545125,1294457825,68921030,3747553711,2523094152,2859947234,3813353925,940551852,1899221899,2339034767,3260459944,420621505,1345212902,3897315384,2708483359,2050271862,856217425,3377582677,2154671986,1529423899,303387964,587282639,1782400488,2977546881,4165320614,35437218,1260439429,2422489324,3646438859,1631206421,671498546,4081239643,3128867708,1076346488,152814431,3529458742,2606971153,2809606523,3997912156,890227509,2083763730,2255139606,3478572593,336742744,1563309183,3846976929,2893039750,1999949807,1040757448,3293689804,2372782827,1445547394,521482405 }
 };
 
-#ifdef HW_CRC
+#ifdef CRC_SSE42
 static const uint32_t crc32c_long[4][256] = {
     { 0,3762348204,3312301993,623728389,2402752931,1870133519,1247456778,2853850790,461522359,4223837467,3740267038,1051660978,2494913556,1962261688,1373170621,2979531537,923044718,3611585474,4066983111,304733291,3090387661,1483960929,2103321956,2636039624,747032281,3435540085,3924523376,162240988,2746341242,1139881942,1725718739,2258403455,1846089436,2387211888,2875498869,1260667353,3778940799,25096147,609466582,3289602170,1972065131,2513155015,2967921858,1353057390,4206643912,452766308,1070726497,3750829517,1494064562,3108863262,2623547931,2082523831,3594159121,913990845,324481976,4078424852,1116723205,2731489449,2279763884,1738706688,3451437478,771236106,148207119,3902117539,3692178872,1012059412,427792913,4181686973,1394947099,3009795255,2521334706,1980261150,3348129807,667977891,50192294,3804053258,1218933164,2833748224,2378876421,1837770409,3944130262,190203514,779817343,3460035027,1679933301,2220974041,2706114780,1091364976,4036289377,282330061,905532616,3585717348,2141452994,2682460782,3134082411,1519300039,2988129124,1381751752,2004257997,2536926305,1026302663,3714892395,4165047662,402748866,2845408979,1238998655,1827981690,2360617430,648963952,3337521116,3821261017,58929269,2233446410,1700745382,1081214883,2687690511,170437033,3932702981,3477413376,788922028,2661152189,2128418065,1542472212,3148915384,296414238,4058647730,3569834935,881310491,3184012673,1569246509,2024118824,2565175940,855585826,3535787150,4153574283,399663911,2789894198,1175095450,1663557535,2204581683,696086933,3376255289,3960522300,206579344,2328667887,1787577923,1335955782,2950820330,100384588,3854262240,3231058149,550955081,2437866328,1896743924,1411601649,3026433117,511310587,4265155159,3675540818,995405310,3552656221,864148465,380407028,4142689368,1559634686,3166093906,2577110359,2044425723,3359866602,671325766,221169987,3983419887,1198810953,2805237733,2182729952,1650012236,3838160947,75845791,564660122,3253266230,1811065232,2343717180,2929663545,1323302549,4282905988,520557864,975847981,3664421505,1886448679,2419068043,3038600078,1432206114,1641676857,2174410901,2763503504,1157060412,4008515994,246282550,729853491,3418377887,2052605326,2585306402,3207983655,1601507979,4117716013,355449985,805497732,3493989160,1457425239,3063835643,2477997310,1945361490,3655963380,967406168,478422365,4240754161,1297927904,2904305228,2284943689,1752275429,3261847363,573257711,117858538,3880157254,265492197,4019451465,3401490764,721306080,2162429766,1621422058,1166653679,2781435971,340874066,4094800894,3510428923,830211159,2607172337,2066131549,1577844056,3192593908,953654667,3633806631,4256836130,502975118,3084944424,1470129284,1921856385,2462962477,592828476,3272947856,3862458261,108564281,2892153247,1277305139,1762620982,2303694490 },
     { 0,2122067443,4244134886,2189719061,4229742909,2187390158,14981851,2123937576,4258657419,2208946552,19771245,2136048286,29963702,2142574661,4247875152,2202878883,4266106855,2151395860,46121985,2093214194,39542490,2082960169,4272096572,2162108623,59927404,2112767647,4285149322,2165710201,4287539793,2180155298,58126775,2097863748,4184033599,2266981580,94048985,2078752554,92243970,2063836657,4186428388,2281438743,79084980,2060095559,4165920338,2251475873,4171922569,2262192506,72492911,2049837724,119854808,2035874603,4225535294,2241896653,4214757349,2235841046,130042883,2042388976,4210106963,2223837088,101284277,2020955206,116253550,2022821533,4195727496,2221512059,4146783375,2304166268,198360937,1974375066,188097970,1967786049,4157505108,2310165415,184487940,1954758135,4127673314,2289788433,4142125369,2292186314,169577183,1952950060,158169960,1997624987,4120191118,2347306365,4117871189,2332923814,160031155,2012597312,4105736163,2328142352,138466309,1983707638,144985822,1993892653,4099675448,2317367499,239709616,1882554435,4071749206,2362301349,4065684621,2351513982,246232939,1892752024,4087109947,2380297416,258212573,1897410350,260085766,1912387061,4084777952,2365910547,4027462231,2389843876,217475505,1921664066,202568554,1919868569,4041910412,2392229247,232507100,1940384559,4045643066,2405413065,4056352737,2411407890,222256135,1933799924,3954819055,2512753180,390194185,1765657082,396721874,1775850273,3948750132,2501970119,376195940,1745905303,3935572098,2498248049,3933243993,2483857322,378065343,1760885836,368975880,1803685371,3909516270,2541377053,3923960117,2543766726,354073299,1801885472,3895186563,2522347888,349409125,1789895318,339154366,1783314509,3905900120,2528338859,316339920,1822874403,3995249974,2422112453,4005967853,2428115486,306080779,1816281592,4010738267,2440245160,334977469,1837855822,320062310,1836052117,4025194624,2442638707,3969739063,2464236740,275067601,1847140130,276932618,1862108665,3967415276,2449858079,289971644,1865723983,3987785306,2479680425,3981728897,2468901234,296486759,1875913364,479419232,1659860627,3765108870,2652319093,3767507549,2666772398,477610427,1644948552,3779506155,2671411736,499053581,1673714174,492465878,1663451941,3785504048,2682132675,3805594759,2628315508,506240865,1615901330,516425146,1622419529,3794820700,2622256047,520171532,1635589631,3824774122,2642757145,3810390321,2640436418,535145175,1637451556,3857715807,2609659820,419973561,1735681098,434951010,1737555601,3843328132,2607326583,405137108,1717160743,3839737138,2594279617,3828951017,2588215834,415333391,1723683324,465014200,1707843659,3880769118,2570320813,3886763141,2581029238,458430307,1697594000,3865213235,2552124608,446309077,1692798758,444512270,1677891069,3867599848,2566573595 },
@@ -29,4 +29,4 @@ static const uint32_t crc32c_short[4][256] = {
     { 0,1249898853,2499797706,3732935599,739488613,1718370816,3102495151,4069761226,1478977226,307795887,3436741632,2259121509,1949619631,1045112010,3771414373,2863973888,2957954452,4197530865,615591774,1859051067,2623373041,3592589204,144874555,1121807710,3899239262,2719373883,2090224020,921286897,3296335931,2382752094,1350957809,452594580,1701965273,789454012,4052619027,3153198710,1231183548,52277209,3718102134,2548190483,1029081875,1999207030,2846452185,3822491836,289749110,1530579219,2243615420,3485805529,3577684045,2671825192,1103151751,197080034,4180448040,3008586317,1842573794,665616519,2367178375,3345463266,434611277,1402492200,2701915618,3950249095,905189160,2139874893,3403930546,2157718231,1578908024,342083613,3874515159,2895092146,1913621021,946895736,2462367096,3636153373,104554418,1279569623,3201764893,4104716152,707325143,1616321970,2058163750,819128131,3998414060,2754413961,1455334723,482432038,3259058057,2285811436,579498220,1760919945,3061158438,4228539203,244958089,1155937004,2590384451,3491352614,2945594987,3857565454,997053601,1897014724,2206303502,3388896363,394160068,1560385185,4155855009,3184173508,1665839723,691356430,3685147588,2446922401,1331233038,86437995,534576127,1436748442,2334333237,3244091472,869222554,2041625087,2804984400,3981400885,1207672117,226782288,3540287487,2575011482,1810378320,563601205,4279749786,3043507711,2418786709,3662950640,81011551,1319883322,3157816048,4131881877,684167226,1656250719,3356272479,2188596794,1559446933,378320112,3827242042,2925585759,1893791472,983500693,543493121,1780153700,3030052555,4276428718,209108836,1175014913,2559139246,3539381451,2017618635,842898350,3971844097,2797762916,1414650286,506341579,3232643940,2329004545,4116327500,3206929705,1638256262,735723491,3645480745,2469817932,1303805411,130649222,2910669446,3875716067,964864076,1945983273,2171533795,3406891142,361831209,1609493068,1158996440,258674877,3521839890,2610229879,1762087613,595108824,4260933751,3079094546,489916178,1464628855,2311874008,3283326141,824194167,2069873938,2782910141,4020250584,1523063335,280488770,3459774701,2219313544,1994107202,1017403431,3794029448,2824583917,49249517,1217432968,2517671463,3698290498,788320136,1686322925,3120770370,4034714663,3934705587,2700682966,2121872761,872858652,3331679446,2364183987,1382712860,404060025,2996941177,4175314972,643716019,1814143702,2662466076,3570267001,172875990,1077023155,1069152254,1975912091,2873496884,3778667601,329925787,1507177982,2270537297,3442104116,1738445108,769744977,4083250174,3105784475,1267540561,32690996,3748839579,2500669950,2415344234,3314076431,453564576,1366763973,2749663503,3919280234,924543941,2103745184,3620756640,2645536197,1127202410,156258063,4223922117,2981895840,1866206479,625212522 },
     { 0,633075163,1266150326,1858870893,2532300652,3008635575,3717741786,4163094785,674486313,227028466,1665414047,1191167556,3202092869,2607282846,4121683187,3490713896,1348972626,1977369993,454056932,1051438655,3330828094,3811807973,2382335112,2823059795,2018797691,1575984544,857965517,379090454,4005281559,3405793996,2781631649,2155323770,2697945252,2239010175,3954739986,3456335561,908113864,328942099,2102877310,1491904933,2298263693,2907131222,3280687931,3861948128,504590305,1000905274,1432650839,1893691788,4037595382,3574801709,3151969088,2657406619,1715931034,1140650561,758180908,143333879,3634039007,4246797572,2481775465,3059160754,1316282291,1808738920,84096005,548979166,1148368825,1640579682,252009487,717138388,3533529301,4146500878,2582285155,3159457464,1816227728,1241160267,657884198,42824189,4205754620,3742715175,2983809866,2489493137,336922603,833548848,1600318557,2061048198,2197475463,2806064476,3381476145,3963014890,1009180610,429730329,2001810548,1391116719,2865301678,2406677877,3787383576,3288667843,3837646621,3238404806,2949266603,2322712944,1917976689,1474950570,958786503,480124444,3431862068,3912628975,2281301122,2722238809,1516361816,2145004931,286667758,883803701,3034212175,2439090836,4289563897,3658905890,573943843,126764536,1765956501,1291431502,2632564582,3109178045,3617477840,4062552331,168192010,800955857,1097958332,1690990183,2296737650,2908658345,3281159364,3861475615,504018974,1001475525,1434276776,1892066931,2698426203,2238530176,3953220845,3457853750,909729847,327325164,2102299521,1492483674,3632455456,4248382203,2482320534,3058614605,1315768396,1809251735,85648378,547427873,4038133513,3574264530,3150376127,2658998628,1717489765,1139090878,757677011,143838728,673845206,227670541,1667097696,1189482939,3200637114,2608737633,4122096396,3490301655,1693695,631382564,1265515593,1859504530,2532704403,3008230728,3716279077,4164558590,2018361220,1576421983,859460658,377594345,4003621096,3407453491,2782233438,2154722949,1350461357,1975882358,453610523,1051883968,3331436737,3811198234,2380684151,2824711852,3432503499,3911986448,2279617405,2723923622,1517817767,2143550076,286254097,884216266,3835953378,3240097081,2949901140,2322079375,1917573006,1475355221,960248888,478661091,2633001113,3108740418,3615982383,4064048884,169852917,799296046,1097356355,1691591064,3032723632,2440578411,4290009862,3658460893,573335516,127373831,1767607402,1289779633,1817753711,1239633332,657413081,43296258,4206325507,3742145240,2982184117,2491117934,1147887686,1641059741,253529072,715619883,3531913002,4148118257,2582863004,3158878535,1010763837,428146150,2001265547,1391662672,2865815377,2406165130,3785831655,3290218812,336384020,834086351,1601911714,2059456121,2195916664,2807624355,3381980366,3962509589 }
 };
-#endif /* HW_CRC */
+#endif /* CRC_SSE42 */
diff --git a/src/crc32c.c b/src/crc32c.c
@@ -45,8 +45,10 @@
 
 #include "crc32c.h"
 
+#include "processor.h"
+
 #if defined(_CPU_X86_64_) && !defined(_COMPILER_MICROSOFT_)
-#  define HW_CRC
+#  define CRC_SSE42
 #endif
 
 /* CRC-32C (iSCSI) polynomial in reversed bit order. */
@@ -74,7 +76,6 @@ static uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len)
     const unsigned char *next = (const unsigned char *) buf;
     uint64_t crc;
 
-    /* pthread_once(&crc32c_once_sw, crc32c_init_sw); */
     crc = crci ^ 0xffffffff;
     while (len && ((uintptr_t)next & 7) != 0) {
         crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
@@ -100,7 +101,7 @@ static uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len)
     return (uint32_t)crc ^ 0xffffffff;
 }
 
-#ifdef HW_CRC
+#ifdef CRC_SSE42
 
 /* Apply the zeros operator table to crc. */
 static inline uint32_t crc32c_shift(const uint32_t zeros[][256], uint32_t crc)
@@ -110,15 +111,12 @@ static inline uint32_t crc32c_shift(const uint32_t zeros[][256], uint32_t crc)
 }
 
 /* Compute CRC-32C using the Intel hardware instruction. */
-static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len)
+static uint32_t crc32c_sse42(uint32_t crc, const void *buf, size_t len)
 {
     const unsigned char *next = (const unsigned char *) buf;
     const unsigned char *end;
     uint64_t crc0, crc1, crc2;      /* need to be 64 bits for crc32q */
 
-    /* populate shift tables the first time through */
-    /* pthread_once(&crc32c_once_hw, crc32c_init_hw); */
-
     /* pre-process the crc */
     crc0 = crc ^ 0xffffffff;
 
@@ -142,8 +140,8 @@ static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len)
         end = next + LONG;
         do {
             __asm__("crc32q\t" "(%3), %0\n\t"
-                                        "crc32q\t" LONGx1 "(%3), %1\n\t"
-                                        "crc32q\t" LONGx2 "(%3), %2"
+                    "crc32q\t" LONGx1 "(%3), %1\n\t"
+                    "crc32q\t" LONGx2 "(%3), %2"
                     : "=r"(crc0), "=r"(crc1), "=r"(crc2)
                     : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2));
             next += 8;
@@ -162,8 +160,8 @@ static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len)
         end = next + SHORT;
         do {
             __asm__("crc32q\t" "(%3), %0\n\t"
-                                        "crc32q\t" SHORTx1 "(%3), %1\n\t"
-                                        "crc32q\t" SHORTx2 "(%3), %2"
+                    "crc32q\t" SHORTx1 "(%3), %1\n\t"
+                    "crc32q\t" SHORTx2 "(%3), %2"
                     : "=r"(crc0), "=r"(crc1), "=r"(crc2)
                     : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2));
             next += 8;
@@ -198,48 +196,34 @@ static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len)
     return (uint32_t)crc0 ^ 0xffffffff;
 }
 
-/* Check for SSE 4.2.  SSE 4.2 was first supported in Nehalem processors
-   introduced in November, 2008.  This does not check for the existence of the
-   cpuid instruction itself, which was introduced on the 486SL in 1992, so this
-   will fail on earlier x86 processors.  cpuid works on all Pentium and later
-   processors. */
-#define SSE42(have) \
-    do { \
-        uint32_t eax, ecx; \
-        eax = 1; \
-        __asm__("cpuid" \
-                : "=c"(ecx) \
-                : "a"(eax) \
-                : "%ebx", "%edx"); \
-        (have) = (ecx >> 20) & 1; \
-    } while (0)
-
-static int sse42 = 0;
-
-#endif /* ifdef HW_CRC */
-
-/* jl_crc32c_init must be called before jl_crc32c.  Passing 1
-   can be used to force the use of the software implementation,
-   which is useful for testing purposes. */
+#endif /* ifdef CRC_SSE42 */
+
+static uint32_t crc32c_dispatch(uint32_t crc, const void *buf, size_t len);
+
+static uint32_t (*crc32c_func)(uint32_t crc, const void *buf, size_t len) = crc32c_dispatch;
+
+// This is exported for test only
 JL_DLLEXPORT void jl_crc32c_init(int force_sw)
 {
-#ifdef HW_CRC
-    if (force_sw)
-        sse42 = 0; /* useful for testing purposes */
-    else
-        SSE42(sse42);
+#ifdef CRC_SSE42
+    if (jl_test_cpu_feature(JL_X86_sse42)) {
+        crc32c_func = crc32c_sse42;
+        return;
+    }
 #endif
+    crc32c_func = crc32c_sw;
 }
 
-/* Compute a CRC-32C.  If the crc32 instruction is available, use the hardware
-   version.  Otherwise, use the software version. */
+static uint32_t crc32c_dispatch(uint32_t crc, const void *buf, size_t len)
+{
+    jl_crc32c_init(0);
+    return crc32c_func(crc, buf, len);
+}
+
+/* Compute a CRC-32C. Pick the best implementation lazily on first call. */
 JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const void *buf, size_t len)
 {
-    return
-#ifdef HW_CRC
-    sse42 ? crc32c_hw(crc, buf, len) :
-#endif
-    crc32c_sw(crc, buf, len);
+    return crc32c_func(crc, buf, len);
 }
 
 /*****************************************************************************/
@@ -398,10 +382,10 @@ int main(void)
     crc32c_init_sw();
     print_array("crc32c_table", 8, 256, &crc32c_table[0][0]);
     crc32c_init_hw();
-    printf("\n#ifdef HW_CRC\n");
+    printf("\n#ifdef CRC_SSE42\n");
     print_array("crc32c_long", 4, 256, &crc32c_long[0][0]);
     print_array("crc32c_short", 4, 256, &crc32c_short[0][0]);
-    printf("#endif /* HW_CRC */\n");
+    printf("#endif /* CRC_SSE42 */\n");
     return 0;
 }
 
diff --git a/src/crc32c.h b/src/crc32c.h
@@ -9,7 +9,6 @@
 extern "C" {
 #endif
 
-JL_DLLEXPORT void jl_crc32c_init(int force_sw);
 JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const void *buf, size_t len);
 
 #ifdef __cplusplus
diff --git a/src/init.c b/src/init.c
@@ -5,6 +5,7 @@
   system initialization and global state
 */
 #include "platform.h"
+#include "crc32c.h"
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/src/support/Makefile b/src/support/Makefile
@@ -7,7 +7,7 @@ override CXXFLAGS += $(JCXXFLAGS)
 override CPPFLAGS += $(JCPPFLAGS)
 
 SRCS := hashing timefuncs ptrhash operators utf8 ios htable bitvector \
-	int2str libsupportinit arraylist strtod crc32c
+	int2str libsupportinit arraylist strtod
 ifeq ($(OS),WINNT)
 SRCS += asprintf wsasocketpair strptime
 ifeq ($(ARCH),i686)
diff --git a/src/support/libsupport.h b/src/support/libsupport.h
diff --git a/src/support/libsupportinit.c b/src/support/libsupportinit.c
