Skip to content

Update Daft to 0.4.13 #362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Update Daft to 0.4.13 #362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d9a1be4
Update to latest version
srilman May 6, 2025
b90f8f0
Update timings
srilman May 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ We also introduced the [Hardware Benchmark](https://benchmark.clickhouse.com/har
- [x] Tablespace
- [x] Tembo
- [x] Cloudberry
- [ ] Daft
- [x] Daft
- [x] Pandas
- [x] Polars
- [x] OctoSQL
Expand Down
4 changes: 2 additions & 2 deletions daft-parquet/benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sudo apt-get update
sudo apt-get install -y python3-pip
pip install --break-system-packages pandas
pip install --break-system-packages packaging
pip install --break-system-packages daft==0.4.9
pip install --break-system-packages daft==0.4.13

# Use for Daft (Parquet, partitioned)
seq 0 99 | xargs -P100 -I{} bash -c 'wget --continue https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet'
Expand All @@ -28,4 +28,4 @@ wget --continue https://datasets.clickhouse.com/hits_compatible/athena/hits.parq
for mode in partitioned single; do
echo "Running $mode mode..."
./run.sh $machine_name $mode 2>&1 | tee "daft_log_${mode}.txt"
done
done
90 changes: 45 additions & 45 deletions daft-parquet/results/c6a.4xlarge.partitioned.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"system": "Daft (Parquet, partitioned)",
"date": "2025-04-10",
"date": "2025-05-06",
"machine": "c6a.4xlarge, 500gb gp2",
"cluster_size": 1,
"comment": "",
Expand All @@ -13,48 +13,48 @@
"load_time": 0,
"data_size": 14779976446,
"result": [
[2.9239969809987088,0.24281721499937703,0.25061354499848676],
[2.943689669999003,0.24701748600091378,0.2412866730010137],
[3.3013779040011286,0.3499440380001033,0.34748943700105883],
[0.5240946279991476,0.14416759599953366,0.14079303099970275],
[8.358806038000694,8.191848701000708,8.301089013999444],
[7.3857230829999025,7.083538949000285,6.779509707001125],
[0.44852974800051015,0.09936009000011836,0.09527912199882849],
[2.9437991849990794,0.24986934899970947,0.249939440000162],
[3.2667164159993263,2.9141958559994237,2.997278159000416],
[6.341243853001288,2.2626934529998834,2.3228578489997744],
[1.1101171179998346,0.8267360560002999,0.7852608660014084],
[1.1224606880005012,0.8311700009999186,0.799573213000258],
[5.719661522000024,2.2495632319987635,2.2330354629993963],
[3.9945796060001157,2.910399188000156,2.8959377400005906],
[5.794290511999861,2.249771316000988,2.2831605970004603],
[6.017177588000777,2.6195074140014185,2.6053394909995404],
[9.390984196999852,5.380491128000358,5.675109460000385],
[7.000278958999843,2.909169999000369,2.9183442980011023],
[10.517328523001197,9.270337152998763,9.211842022999917],
[0.4838494970008469,0.13263488300071913,0.13043909699990763],
[13.509324963999461,2.8007291499998246,2.7599115609991713],
[15.182025014000828,3.0815161779992195,3.0070982840006764],
[25.465576216000045,7.294880023000587,7.23245921400121],
[55.77252582199981,14.482481478000409,14.470004283999515],
[5.588168801001302,3.3266251870009,3.264028256000529],
[11.422374611000123,11.096233861000655,11.42879000799985],
[8.442051498999717,6.437731754000197,6.3204276689994],
[13.50050955000006,4.382914202000393,4.389906216998497],
[54.93292986199958,54.7095186509996,54.55201650199888],
[1.810280910000074,1.694118317000175,1.718246561000342],
[6.6975024990006204,1.7159326830005739,1.6963492020004196],
[7.2224263429998246,2.2286510949998046,2.2376933390005433],
[14.639526433000356,12.763357517000259,12.641049649000706],
[20.836833830000614,12.953915013998994,13.191902100001244],
[19.975888697999835,12.617277572000603,12.521102639999299],
[2.9711594529999275,2.7580122080016736,2.77822500900038],
[0.7095541710004909,0.3223742640002456,0.2954207420007151],
[0.5843301879995124,0.15686079099941708,0.14083533799930592],
[0.6433542939994368,0.16117779100022744,0.14962360200115654],
[0.9204736569990928,0.4168329120002454,0.4444124349993217],
[0.5595925659999921,0.10117537899895979,0.09999015900029917],
[0.5403744819996064,0.09798703699925682,0.09515768100027344],
[0.5390828099989449,0.08939602699865645,0.08288451900079963]
[3.0129025579999507,0.24875987399991573,0.2496312430002945],
[2.9242310099998576,0.24344808199975887,0.24112270199975683],
[3.2836581950000436,0.34658900300019013,0.3432621920001111],
[0.632236310999815,0.14180675299985523,0.14148228500016558],
[8.226634456999818,8.367781668999669,8.18395400999998],
[7.238864371000091,6.957451859999765,7.0355879319999985],
[0.46573037200005274,0.1015544349997981,0.09943150900016917],
[2.9443942260004405,0.246822369000256,0.2457495150001705],
[3.167501248999997,2.9036229090002053,2.928460234999875],
[6.278120196000145,2.306866156000069,2.3286495050001577],
[1.0777284720002172,0.8103329299997313,0.8184506249999686],
[1.1107370419999825,0.826304593000259,0.8324333659998047],
[5.684191222000209,2.1915008070000113,2.2823265170000013],
[3.9283386309998605,2.8848196290000487,2.869468936999965],
[5.796531786999822,2.191972125999655,2.231937829999879],
[5.963768410000284,2.6110957099999723,2.6199507679998533],
[9.3568840490002,5.2695333530000426,5.288399773000037],
[6.979389613999956,2.8898374800000965,2.8538799349998953],
[10.348535809000168,9.22630863500035,9.246704696999586],
[0.4559227129998362,0.1325727450002887,0.13311697900007857],
[13.465921836999769,2.771887383000376,2.762853549999818],
[15.173226090000298,3.000628551999853,3.024777911000001],
[25.466720822999832,7.140110374999949,7.121832840999559],
[55.710079480999866,14.442347977999816,14.522546690000127],
[5.786835220000285,3.623697769000046,3.4196181370002705],
[11.31074052799977,11.037521419000313,11.36143081299997],
[8.532309114000327,6.517149922000044,6.295931412999835],
[13.51063966599986,4.312364394999804,4.281718726000236],
[55.159144451999964,54.98963902599962,55.00981655999976],
[2.0328813870000886,1.9355504010000004,1.938292840000031],
[6.674335584999881,1.682719822999843,1.679446862000077],
[7.207933225000033,2.2170512609995967,2.2262567170000693],
[14.678110961999664,12.464046565999979,12.826411793999796],
[20.782259919999888,12.862545865999891,12.892152691999854],
[20.008948390000114,11.882019637000212,12.018335267000111],
[2.902464766999856,2.7109572730000764,2.753635456000211],
[0.6867136749997371,0.29985155800022767,0.304267021999749],
[0.5178484460002437,0.15791345400020873,0.14455797900018297],
[0.6576429560000179,0.16304248400001597,0.16121927499989397],
[0.9528694539999378,0.457950745999824,0.4238556980003523],
[0.5734661550000055,0.10209436100012681,0.09967286999972202],
[0.565142668000135,0.09541054999999687,0.09478621499965811],
[0.5354012520001561,0.08874468700014404,0.08454601699986597]
]
}
}
90 changes: 45 additions & 45 deletions daft-parquet/results/c6a.4xlarge.single.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"system": "Daft (Parquet, single)",
"date": "2025-04-10",
"date": "2025-05-06",
"machine": "c6a.4xlarge, 500gb gp2",
"cluster_size": 1,
"comment": "",
Expand All @@ -13,48 +13,48 @@
"load_time": 0,
"data_size": 14779976446,
"result": [
[3.7661174910008413,0.24369674700028554,0.2084103370016237],
[3.875728433000404,0.2710822070002905,0.2448596260001068],
[4.437644478999573,0.37059677300021576,0.3738455969996721],
[1.7555348349997075,0.1765103719990293,0.16613175199927355],
[9.246005683000476,8.021207697000136,7.9671034419989155],
[8.395427339999515,6.894040901999688,6.937520977000531],
[0.3870741339997039,0.12874933700004476,0.11846869800137938],
[4.032850421999683,0.27275637599996116,0.27422388899867656],
[4.48173838799994,2.9867709099999047,2.9398520889990323],
[7.820063309000034,2.3338501990001532,2.297853408001174],
[2.550961462000487,0.831118198000695,0.8376970059998712],
[2.6340099579992966,0.8703369129998464,0.8628015900012542],
[6.752234794999822,2.222958735999782,2.232471744000577],
[5.255753490000643,2.9050689529994997,2.930579284000487],
[6.612870382001347,2.2050894149997475,2.23886461300026],
[6.808695531000922,2.6192918489996373,2.6568191360001947],
[10.58538459600095,5.351642343999629,5.352705452000009],
[8.140032625000458,2.8994017459990573,2.8980922949995147],
[11.584158911999111,9.256097968000176,9.276376194000477],
[1.5025645540008554,0.19396826999945915,0.17483294299927366],
[13.306670707999729,2.671392956001,2.641569294000874],
[14.934801787001561,2.7154280849990755,2.688567492999937],
[25.46551972100133,5.02428535099898,4.921547977000955],
[63.64726643300128,24.418693142999473,24.98236643400014],
[6.463048370998877,3.3630810270005895,3.1713167640009488],
[12.681365424999967,10.957809773999543,11.052705301999595],
[9.517097973999626,6.553240066999933,6.373768290999578],
[13.353289077998852,4.344104071999027,4.224701305000053],
[55.07765850400028,54.390547955999864,54.564625152999724],
[1.9125421499993536,1.7788892349999514,1.8072454829998605],
[8.256591348999791,1.7278820299998188,1.7305858340005216],
[8.80483117300173,2.2188438910015975,2.2703534689990192],
[16.096305291001045,13.049876507999215,12.798272336000082],
[20.626275730000998,12.63391932200102,12.84786659099882],
[19.942403981998723,12.12704556800054,12.179334143000233],
[3.1018452330008586,2.8029017110002314,2.8341677140015236],
[0.7097889510005189,0.33531982000022253,0.31982642600087274],
[0.5140427599999384,0.1496123090000765,0.13467889399908017],
[0.6331659280003805,0.1702573789989401,0.1500775069998781],
[1.011886082998899,0.42512062800051353,0.4513590800015663],
[0.4912343949999922,0.1039613590000954,0.08932384900072066],
[0.46491453100134095,0.08819261000098777,0.08146828899953107],
[0.38542473499910557,0.08088228899941896,0.07185498099897814]
[3.2253568189998987,0.23140306699997382,0.21925035000003845],
[3.970349201999852,0.2605729239999164,0.25809068100011245],
[4.5416300800000045,0.37985720599999695,0.356041335999862],
[1.7193743330001325,0.16728201699993406,0.17433243599998605],
[9.712110386000177,8.211728058999824,7.884640712999953],
[8.518551183999989,7.488730011000143,6.803818503999992],
[0.387745754999969,0.12840621999998802,0.11804172200004359],
[3.9401003340001353,0.2691451910000069,0.2730321340000046],
[4.414285607000011,3.00756264000006,2.999268239999992],
[7.6969749499999125,2.305176566,2.323771593999936],
[2.4990377849999277,0.8528864209999938,0.8339026910000484],
[2.5852065640001456,0.8578816259998803,0.8560670679999021],
[6.566081167999982,2.1586584980000225,2.22068322899986],
[5.243936185999928,2.8671286429998872,2.8879154630001267],
[6.6861919359998865,2.1887674380000135,2.1895471739999266],
[6.992311629999904,2.610773949999839,2.6136060149999594],
[10.410779798000021,5.379705058999889,5.307476756999904],
[8.03119067900002,2.8503332609998324,2.882438097999966],
[11.571397067999897,8.942628501999934,9.1885218110001],
[1.565434324999842,0.1849234549999892,0.18494166599998607],
[13.318500523000012,2.6387486019998505,2.575133064000056],
[15.07152691300007,2.6578845770000044,2.641332942999952],
[25.468179423000038,4.916547996999952,4.888193249000096],
[63.255035464,24.551993990000028,24.946234153999967],
[6.433144932000005,3.296971314000075,3.1965741799999705],
[12.790251215999888,11.350317142999984,10.955297184999836],
[9.594065618000059,6.336623910000071,6.492794157999924],
[13.44900045899999,4.262191568000162,4.275473608000084],
[55.55805890400006,55.16661785800011,55.17273365400001],
[1.906259638000165,1.7596941069998593,1.7414089720000447],
[8.304442497999844,1.7121706929999618,1.685062291000122],
[8.905798615999856,2.205666161000181,2.2117088380000496],
[16.040822982999998,12.61951165000005,12.637090342999954],
[20.71301297800005,12.760998825999877,12.609494229999882],
[19.673526806999917,11.59993482599998,11.916693481000038],
[3.0532797200000914,2.7271483160000116,2.7536142059998383],
[0.7373581440001544,0.3060946909999984,0.2917485090001719],
[0.49712218300010136,0.1518116749998626,0.15398914200000036],
[0.6466036060000988,0.1693926579998788,0.16669720200002303],
[0.9969094470000073,0.41466951500001414,0.4561509259999639],
[0.49488785299990923,0.10238816000014594,0.08770358099991427],
[0.4575332649999382,0.08703326699992431,0.08424600800003645],
[0.3789832980000938,0.07373583599996891,0.06858976699982122]
]
}
}
90 changes: 45 additions & 45 deletions daft-parquet/results/c6a.metal.partitioned.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"system": "Daft (Parquet, partitioned)",
"date": "2025-04-10",
"date": "2025-05-07",
"machine": "c6a.metal, 500gb gp2",
"cluster_size": 1,
"comment": "",
Expand All @@ -13,48 +13,48 @@
"load_time": 0,
"data_size": 14779976446,
"result": [
[2.817390483000054,0.13732489699987127,0.13498625000011089],
[2.8280520220000653,0.14926322500014066,0.14125483799989524],
[3.186676187000103,0.18360701199981122,0.17821930999980395],
[0.5109683139999106,0.11087876300007338,0.11180828299984569],
[7.937879661999887,7.251060402999883,7.168233010000222],
[6.645456091000142,6.241062057999898,6.157182824999836],
[0.45029773800024486,0.097797453000112,0.11483498899997358],
[2.836866591999751,0.12852769299979627,0.12009974700004022],
[2.575397141999929,2.0285684870000296,2.0237227059997167],
[5.806728040000053,1.408875776000059,1.458710291999978],
[1.052088324000124,0.6057662680000249,0.6002813199997945],
[1.0623894289997224,0.5801979930001835,0.5796440919998531],
[5.310391408999749,1.3735298459996557,1.237582190000012],
[3.2555085549997784,1.5031512900000052,1.4484107289999884],
[5.435526366999966,1.423710137999933,1.2569544040002256],
[5.221850785999777,1.2954923000002054,1.272231783999814],
[8.47660143999974,3.796346491999884,3.197499517000324],
[6.082066434999888,0.6127163279998058,0.6317263159999129],
[8.143912356000328,4.904919783999958,4.928797715999735],
[0.5063426940000681,0.10949358799962283,0.10352339199971539],
[13.30107757099995,1.4053640460001589,1.2949443400002565],
[14.954863689999911,1.5408073439998589,1.4785505589998138],
[25.205442676000075,3.005651143999785,2.416006015000221],
[55.82200932099977,3.3242247089997363,3.22857743700024],
[5.230610652999985,2.607239872000264,2.5258993530001135],
[10.25858594400006,9.482845454000199,9.646128224999757],
[7.894359932000043,5.291036427999643,5.289886783999918],
[13.352615854000305,1.9040649989997291,1.5829486399998132],
[12.728114092999931,6.617246512999827,6.583521141999881],
[0.5924163849999786,0.28892527199968754,0.26512917400032165],
[6.3112071319997085,0.7529843540000911,0.7514655820000371],
[6.627713015000154,1.1418134529999406,1.1509752590000062],
[10.132989218999683,6.204255733000082,6.747729526000057],
[18.396693426000184,7.266776995000328,7.7042247620001945],
[18.248295537000104,7.222486666999885,7.241226202999769],
[1.4616745640000772,1.1589692660004403,1.047514640999907],
[0.7227331760000197,0.27875184899994565,0.2914327729999968],
[0.568182509000053,0.15323798000008537,0.15695394000022134],
[0.6388387289998718,0.16867721399967195,0.1590756959999453],
[0.9111694060002264,0.39668368100001317,0.38005373399983],
[0.533883331999732,0.10714776400027404,0.10720637599979455],
[0.5362726539997311,0.1020494049998888,0.1014849929997581],
[0.515385674999834,0.10121104700010619,0.09968363400003]
[4.542686241000183,0.3400378350002029,0.34052513899996484],
[4.717193322999947,0.3159660410001379,0.33263838099992427],
[5.169724720999966,0.3057518240000263,0.33807921399989027],
[1.8219968170001266,0.1398865639998803,0.13099126700012675],
[9.221521310000071,7.372719621999977,7.613359848999835],
[8.029815018000136,6.088734087000148,6.054600596],
[0.3797095420000005,0.08547698099982881,0.07792622099987057],
[4.920081631999892,0.2557779430001119,0.23739429299985204],
[4.072021078000034,2.0438079090001793,2.018075637000038],
[8.821021174999942,1.559356242000149,1.6399656690000484],
[2.824433024999962,0.591188246999991,0.6036700560000554],
[2.909164111000109,0.5846963460001007,0.5893974970001636],
[7.638556075999986,1.5113437340000928,1.309223571000075],
[5.236202391000006,1.5070262750000438,1.4533289139999397],
[7.863551977000043,1.336604940999905,1.501996821000148],
[7.414907213000106,1.4089349560001665,1.4150522649999857],
[10.882626166000136,3.8392515139998977,3.3979335389999505],
[8.848038450999866,0.9335955080000531,0.8020202520001476],
[10.906040935999954,5.302365182999893,5.162245837],
[1.762066508000089,0.15736897800002225,0.15247489300008965],
[17.05695661699997,1.3736198840001634,1.2905166929999723],
[18.154430534000085,1.5832041490000393,1.4380508910001026],
[31.479675945000054,1.8425098709999475,1.7850351170000067],
[65.79662893699992,7.4705828699998165,7.53095445200006],
[7.201177490999953,3.180423244999929,3.31553825900005],
[11.721691548999843,9.352431253000077,9.841703271000142],
[9.894703149999941,5.296786537999878,5.406372586000089],
[18.442065364999962,1.530674154000053,1.3862329839998893],
[17.252178706999985,6.681394331000092,6.888012864999837],
[0.9683775650000825,0.346284002999937,0.3117288970001937],
[9.299956111000029,1.030356238999957,0.9512034299998504],
[9.770419508000032,1.2958980610001163,1.3564758129998609],
[13.036262571999941,6.115226422999967,6.129954652999913],
[22.87313654400009,7.648816832000193,7.447812742999986],
[23.234210738,7.826975045999916,7.105942515999914],
[2.5784665149999455,1.2015179070001523,1.078291123000099],
[0.8301606849997825,0.3023212670000248,0.2803503669999827],
[0.5624936850001632,0.1470868490000612,0.1272203679998256],
[0.7277017050000723,0.156387771000027,0.15528855899992777],
[1.099804929999891,0.3980845880000743,0.43361095000000205],
[0.5348942260000058,0.09172149599999102,0.09614963100011664],
[0.5258316580000155,0.0924055049999879,0.08599235300016517],
[0.4086826610000571,0.07297354299998915,0.06994259700013572]
]
}
}
Loading