Skip to content

Commit 33c6fac

Browse files
craig[bot]andy-kimball
craig[bot]
and
andy-kimball
committed
Merge #27046
27046: opt: Improve FuncDepSet r=andy-kimball a=andy-kimball This PR contains a number of improvements and fixes to the functional dependency library: - Normalize constants and equivalencies so there's less redundancy - Allow MakeOuter dependencies that cross join boundaries - Make ProjectCols actually remove un-projected columns - Fix FDs generated for apply joins - Add Verify method to FuncDepSet Together, these changes clean up the FD logical props so that they're more compact and more efficient. Co-authored-by: Andrew Kimball <andyk@cockroachlabs.com>
2 parents 94524bf + 52dff41 commit 33c6fac

File tree

40 files changed

+1337
-1018
lines changed

40 files changed

+1337
-1018
lines changed

‎pkg/sql/opt/memo/logical_props_builder.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,11 @@ func (b *logicalPropsBuilder) buildJoinProps(ev ExprView) props.Logical {
364364
// 2. Apply ON predicate filter on resulting rows.
365365
// 3. For outer joins, add non-matching rows, extended with NULL values
366366
// for the null-supplying side of the join.
367-
relational.FuncDeps.MakeProduct(&rightProps.FuncDeps)
367+
if ev.IsJoinApply() {
368+
relational.FuncDeps.MakeApply(&rightProps.FuncDeps)
369+
} else {
370+
relational.FuncDeps.MakeProduct(&rightProps.FuncDeps)
371+
}
368372

369373
notNullCols := leftProps.NotNullCols.Union(rightProps.NotNullCols)
370374

‎pkg/sql/opt/memo/testdata/logprops/constraints‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ scan c@v
678678
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
679679
├── cost: 0.00648979592
680680
├── key: (1)
681-
├── fd: (1)-->(2,3), ()-->(3)
681+
├── fd: ()-->(3), (1)-->(2)
682682
├── prune: (1)
683683
└── interesting orderings: (+1) (+3,+2,+1)
684684

@@ -691,7 +691,7 @@ scan c@v
691691
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
692692
├── cost: 0.00648979592
693693
├── key: (1)
694-
├── fd: (1)-->(2,3), ()-->(2)
694+
├── fd: ()-->(2), (1)-->(3)
695695
├── prune: (1)
696696
└── interesting orderings: (+1) (+3,+2,+1)
697697

@@ -717,7 +717,7 @@ scan c@v
717717
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
718718
├── cost: 0.00648979592
719719
├── key: (1)
720-
├── fd: (1)-->(2,3), ()-->(3)
720+
├── fd: ()-->(3), (1)-->(2)
721721
├── prune: (1)
722722
└── interesting orderings: (+1) (+3,+2,+1)
723723

@@ -730,7 +730,7 @@ scan c@v
730730
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
731731
├── cost: 0.00648979592
732732
├── key: (1)
733-
├── fd: (1)-->(2,3), ()-->(2)
733+
├── fd: ()-->(2), (1)-->(3)
734734
├── prune: (1)
735735
└── interesting orderings: (+1) (+3,+2,+1)
736736

@@ -877,7 +877,7 @@ select
877877
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
878878
├── cost: 1070
879879
├── key: (1)
880-
├── fd: (1)-->(2,3), ()-->(2)
880+
├── fd: ()-->(2), (1)-->(3)
881881
├── prune: (1)
882882
├── interesting orderings: (+1)
883883
├── scan d
@@ -912,7 +912,7 @@ select
912912
├── stats: [rows=0.00612244898, distinct(2)=0.00612244898, distinct(3)=0.00612244898]
913913
├── cost: 1070
914914
├── key: (1)
915-
├── fd: (1)-->(2,3), ()-->(3)
915+
├── fd: ()-->(3), (1)-->(2)
916916
├── prune: (1)
917917
├── interesting orderings: (+1)
918918
├── scan d

‎pkg/sql/opt/memo/testdata/logprops/groupby‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,9 @@ project
9191
├── grouping columns: z:3(float!null) s:4(string)
9292
├── stats: [rows=1000, distinct(3,4)=1000]
9393
├── key: (3,4)
94-
├── fd: (1)-->(3,4), (3,4)~~>(1) [removed: (1)]
9594
└── project
9695
├── columns: z:3(float!null) s:4(string)
9796
├── stats: [rows=1000, distinct(3,4)=1000]
98-
├── fd: (1)-->(3,4), (3,4)~~>(1) [removed: (1)]
9997
└── scan xyzs
10098
├── columns: x:1(int!null) y:2(int) z:3(float!null) s:4(string)
10199
├── stats: [rows=1000, distinct(3,4)=1000]
@@ -140,7 +138,6 @@ group-by
140138
├── project
141139
│ ├── columns: z:3(float!null) s:4(string)
142140
│ ├── stats: [rows=1000, distinct(3)=700]
143-
│ ├── fd: (1)-->(3,4), (3,4)~~>(1) [removed: (1)]
144141
│ └── scan xyzs
145142
│ ├── columns: x:1(int!null) y:2(int) z:3(float!null) s:4(string)
146143
│ ├── stats: [rows=1000, distinct(3)=700]

‎pkg/sql/opt/memo/testdata/logprops/index-join‎

Lines changed: 54 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,28 +24,50 @@ TABLE a
2424
├── s string
2525
└── x int not null (storing)
2626

27-
# In order to actually create new logical props for the lookup join, we
28-
# need to call ConstructLookupJoin, which only happens when where is a
29-
# remaining filter.
27+
exec-ddl
28+
CREATE TABLE abc (
29+
a INT,
30+
b INT,
31+
c INT,
32+
d INT,
33+
PRIMARY KEY (a, b),
34+
INDEX (c)
35+
)
36+
----
37+
TABLE abc
38+
├── a int not null
39+
├── b int not null
40+
├── c int
41+
├── d int
42+
├── INDEX primary
43+
│ ├── a int not null
44+
│ └── b int not null
45+
└── INDEX secondary
46+
├── c int
47+
├── a int not null
48+
└── b int not null
49+
50+
# In order to actually create new logical props for the index join, we need to
51+
# call ConstructLookupJoin, which only happens when where is a remaining filter.
3052
opt
3153
SELECT * FROM a WHERE s = 'foo' AND x + y = 10
3254
----
3355
select
3456
├── columns: x:1(int!null) y:2(int) s:3(string!null) d:4(decimal!null)
3557
├── stats: [rows=0.476190476, distinct(3)=0.476190476]
3658
├── key: (1)
37-
├── fd: (1)-->(2-4), (3,4)-->(1,2), (2,3)~~>(1,4), ()-->(3)
59+
├── fd: ()-->(3), (1)-->(2,4), (4)-->(1,2), (2,3)~~>(1,4)
3860
├── index-join a
3961
│ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null)
4062
│ ├── stats: [rows=1.42857143]
4163
│ ├── key: (1)
42-
│ ├── fd: (1)-->(2-4), (3,4)-->(1), ()-->(3), (3,4)~~>(1,2), (2,3)~~>(1,4)
64+
│ ├── fd: ()-->(3), (1)-->(2,4), (4)-->(1), (3,4)~~>(1,2), (2,3)~~>(1,4)
4365
│ └── scan a@secondary
4466
│ ├── columns: x:1(int!null) s:3(string!null) d:4(decimal!null)
4567
│ ├── constraint: /-3/4: [/'foo' - /'foo']
4668
│ ├── stats: [rows=1.42857143, distinct(3)=1]
4769
│ ├── key: (1)
48-
│ └── fd: (1)-->(3,4), (3,4)-->(1), ()-->(3)
70+
│ └── fd: ()-->(3), (1)-->(4), (4)-->(1)
4971
└── filters [type=bool, outer=(1,2)]
5072
└── eq [type=bool, outer=(1,2)]
5173
├── plus [type=int, outer=(1,2)]
@@ -63,12 +85,12 @@ project
6385
├── columns: x:1(int!null) y:2(int) s:3(string!null)
6486
├── stats: [rows=0.476190476, distinct(3)=0.476190476]
6587
├── key: (1)
66-
├── fd: (1)-->(2,3), (2,3)~~>(1), ()-->(3)
88+
├── fd: ()-->(3), (1)-->(2), (2,3)~~>(1)
6789
├── index-join a
6890
│ ├── columns: x:1(int!null) y:2(int) s:3(string)
6991
│ ├── stats: [rows=1.42857143]
7092
│ ├── key: (1)
71-
│ ├── fd: ()-->(3), (1)-->(2,3), (2,3)~~>(1)
93+
│ ├── fd: ()-->(3), (1)-->(2), (2,3)~~>(1)
7294
│ └── scan a@secondary
7395
│ ├── columns: x:1(int!null) s:3(string!null)
7496
│ ├── constraint: /-3/4: [/'foo' - /'foo']
@@ -81,3 +103,27 @@ project
81103
│ ├── variable: a.x [type=int, outer=(1)]
82104
│ └── variable: a.y [type=int, outer=(2)]
83105
└── const: 10 [type=int]
106+
107+
# Use secondary index to join to multi-valued primary index, but project only
108+
# a subset of the primary columns.
109+
opt
110+
SELECT b, c, d FROM abc WHERE c=1 AND d=2
111+
----
112+
select
113+
├── columns: b:2(int!null) c:3(int!null) d:4(int!null)
114+
├── stats: [rows=0.00204081633, distinct(3)=0.00204081633, distinct(4)=0.00204081633]
115+
├── fd: ()-->(3,4)
116+
├── index-join abc
117+
│ ├── columns: b:2(int!null) c:3(int) d:4(int)
118+
│ ├── stats: [rows=1.42857143]
119+
│ ├── fd: ()-->(3)
120+
│ └── scan abc@secondary
121+
│ ├── columns: a:1(int!null) b:2(int!null) c:3(int!null)
122+
│ ├── constraint: /3/1/2: [/1 - /1]
123+
│ ├── stats: [rows=1.42857143, distinct(3)=1]
124+
│ ├── key: (1,2)
125+
│ └── fd: ()-->(3)
126+
└── filters [type=bool, outer=(4), constraints=(/4: [/2 - /2]; tight), fd=()-->(4)]
127+
└── eq [type=bool, outer=(4), constraints=(/4: [/2 - /2]; tight)]
128+
├── variable: abc.d [type=int, outer=(4)]
129+
└── const: 2 [type=int]

‎pkg/sql/opt/memo/testdata/logprops/join‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,11 @@ SELECT (SELECT (VALUES (x), (y))) FROM xysd
5454
project
5555
├── columns: column1:7(int)
5656
├── stats: [rows=1000]
57-
├── fd: ()-->(5), (5)-->(7) [removed: (5)]
5857
├── inner-join-apply
5958
│ ├── columns: x:1(int!null) y:2(int) column1:5(int)
6059
│ ├── stats: [rows=1000]
6160
│ ├── key: (1)
62-
│ ├── fd: (1)-->(2), ()-->(5)
61+
│ ├── fd: (1)-->(2,5)
6362
│ ├── scan xysd
6463
│ │ ├── columns: x:1(int!null) y:2(int)
6564
│ │ ├── stats: [rows=1000]
@@ -122,7 +121,7 @@ project
122121
├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null) u:5(int!null)
123122
├── stats: [rows=1000]
124123
├── key: (1)
125-
├── fd: (1)-->(2-4), (3,4)~~>(1,2), ()-->(5)
124+
├── fd: (1)-->(2-5), (3,4)~~>(1,2)
126125
├── scan xysd
127126
│ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null)
128127
│ ├── stats: [rows=1000]
@@ -146,11 +145,10 @@ project
146145
│ │ ├── columns: u:5(int!null)
147146
│ │ ├── outer: (1)
148147
│ │ ├── stats: [rows=111.111111]
149-
│ │ ├── fd: (5)==(1), (1)==(5) [removed: (1)]
150148
│ │ ├── scan uv
151149
│ │ │ ├── columns: u:5(int)
152150
│ │ │ └── stats: [rows=1000]
153-
│ │ └── filters [type=bool, outer=(1,5), constraints=(/1: (/NULL - ]; /5: (/NULL - ]), fd=(5)==(1), (1)==(5)]
151+
│ │ └── filters [type=bool, outer=(1,5), constraints=(/1: (/NULL - ]; /5: (/NULL - ]), fd=(1)==(5), (5)==(1)]
154152
│ │ └── eq [type=bool, outer=(1,5), constraints=(/1: (/NULL - ]; /5: (/NULL - ])]
155153
│ │ ├── variable: uv.u [type=int, outer=(5)]
156154
│ │ └── variable: xysd.x [type=int, outer=(1)]
@@ -201,12 +199,12 @@ right-join
201199
│ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null) uv.u:8(int)
202200
│ ├── stats: [rows=1000, distinct(8)=1]
203201
│ ├── key: (1)
204-
│ ├── fd: (1)-->(2-4,8), (3,4)~~>(1,2), ()-->(8)
202+
│ ├── fd: ()-->(8), (1)-->(2-4), (3,4)~~>(1,2)
205203
│ ├── left-join-apply
206204
│ │ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null) uv.u:8(int)
207205
│ │ ├── stats: [rows=1000, distinct(8)=1]
208206
│ │ ├── key: (1)
209-
│ │ ├── fd: (1)-->(2-4,8), (3,4)~~>(1,2), ()~~>(8)
207+
│ │ ├── fd: (1)-->(2-4,8), (3,4)~~>(1,2)
210208
│ │ ├── scan xysd
211209
│ │ │ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null)
212210
│ │ │ ├── stats: [rows=1000]
@@ -223,16 +221,14 @@ right-join
223221
│ │ │ ├── columns: uv.u:8(int!null)
224222
│ │ │ ├── outer: (1)
225223
│ │ │ ├── stats: [rows=110.111111]
226-
│ │ │ ├── fd: (8)==(1), (1)==(8) [removed: (1)]
227224
│ │ │ ├── select
228225
│ │ │ │ ├── columns: uv.u:8(int!null)
229226
│ │ │ │ ├── outer: (1)
230227
│ │ │ │ ├── stats: [rows=111.111111]
231-
│ │ │ │ ├── fd: (8)==(1), (1)==(8) [removed: (1)]
232228
│ │ │ │ ├── scan uv
233229
│ │ │ │ │ ├── columns: uv.u:8(int)
234230
│ │ │ │ │ └── stats: [rows=1000]
235-
│ │ │ │ └── filters [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(8)==(1), (1)==(8)]
231+
│ │ │ │ └── filters [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)]
236232
│ │ │ │ └── eq [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ])]
237233
│ │ │ │ ├── variable: uv.u [type=int, outer=(8)]
238234
│ │ │ │ └── variable: xysd.x [type=int, outer=(1)]
@@ -307,16 +303,14 @@ project
307303
│ │ ├── columns: uv.u:8(int!null)
308304
│ │ ├── outer: (1)
309305
│ │ ├── stats: [rows=110.111111]
310-
│ │ ├── fd: (8)==(1), (1)==(8) [removed: (1)]
311306
│ │ ├── select
312307
│ │ │ ├── columns: uv.u:8(int!null)
313308
│ │ │ ├── outer: (1)
314309
│ │ │ ├── stats: [rows=111.111111]
315-
│ │ │ ├── fd: (8)==(1), (1)==(8) [removed: (1)]
316310
│ │ │ ├── scan uv
317311
│ │ │ │ ├── columns: uv.u:8(int)
318312
│ │ │ │ └── stats: [rows=1000]
319-
│ │ │ └── filters [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(8)==(1), (1)==(8)]
313+
│ │ │ └── filters [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)]
320314
│ │ │ └── eq [type=bool, outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ])]
321315
│ │ │ ├── variable: uv.u [type=int, outer=(8)]
322316
│ │ │ └── variable: xysd.x [type=int, outer=(1)]
@@ -335,7 +329,7 @@ semi-join
335329
├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null)
336330
├── stats: [rows=100000]
337331
├── key: (1)
338-
├── fd: (1)-->(2-4), (3,4)~~>(1,2), (1)==(5), (5)==(1) [removed: (5)]
332+
├── fd: (1)-->(2-4), (3,4)~~>(1,2)
339333
├── scan xysd
340334
│ ├── columns: x:1(int!null) y:2(int) s:3(string) d:4(decimal!null)
341335
│ ├── stats: [rows=1000]
@@ -367,16 +361,14 @@ semi-join-apply
367361
│ ├── columns: u:5(int) v:6(int!null)
368362
│ ├── outer: (1)
369363
│ ├── stats: [rows=110.111111]
370-
│ ├── fd: (6)==(1), (1)==(6) [removed: (1)]
371364
│ ├── select
372365
│ │ ├── columns: u:5(int) v:6(int!null)
373366
│ │ ├── outer: (1)
374367
│ │ ├── stats: [rows=111.111111]
375-
│ │ ├── fd: (6)==(1), (1)==(6) [removed: (1)]
376368
│ │ ├── scan uv
377369
│ │ │ ├── columns: u:5(int) v:6(int!null)
378370
│ │ │ └── stats: [rows=1000]
379-
│ │ └── filters [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(6)==(1), (1)==(6)]
371+
│ │ └── filters [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)]
380372
│ │ └── eq [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ])]
381373
│ │ ├── variable: uv.v [type=int, outer=(6)]
382374
│ │ └── variable: xysd.x [type=int, outer=(1)]
@@ -423,16 +415,14 @@ anti-join-apply
423415
│ ├── columns: u:5(int) v:6(int!null)
424416
│ ├── outer: (1)
425417
│ ├── stats: [rows=110.111111]
426-
│ ├── fd: (6)==(1), (1)==(6) [removed: (1)]
427418
│ ├── select
428419
│ │ ├── columns: u:5(int) v:6(int!null)
429420
│ │ ├── outer: (1)
430421
│ │ ├── stats: [rows=111.111111]
431-
│ │ ├── fd: (6)==(1), (1)==(6) [removed: (1)]
432422
│ │ ├── scan uv
433423
│ │ │ ├── columns: u:5(int) v:6(int!null)
434424
│ │ │ └── stats: [rows=1000]
435-
│ │ └── filters [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(6)==(1), (1)==(6)]
425+
│ │ └── filters [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)]
436426
│ │ └── eq [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ])]
437427
│ │ ├── variable: uv.v [type=int, outer=(6)]
438428
│ │ └── variable: xysd.x [type=int, outer=(1)]
@@ -598,12 +588,10 @@ semi-join-apply
598588
│ ├── outer: (1)
599589
│ ├── cardinality: [0 - 5]
600590
│ ├── stats: [rows=5]
601-
│ ├── fd: (1)==(5), (5)==(1) [removed: (1)]
602591
│ ├── select
603592
│ │ ├── columns: u:5(int!null) v:6(int!null)
604593
│ │ ├── outer: (1)
605594
│ │ ├── stats: [rows=111.111111]
606-
│ │ ├── fd: (1)==(5), (5)==(1) [removed: (1)]
607595
│ │ ├── scan uv
608596
│ │ │ ├── columns: u:5(int) v:6(int!null)
609597
│ │ │ └── stats: [rows=1000]
@@ -635,7 +623,7 @@ anti-join
635623
├── scan uv
636624
│ ├── columns: u:2(int) v:3(int!null)
637625
│ └── stats: [rows=1000]
638-
└── filters [type=bool, outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ]), fd=(2)==(1), (1)==(2)]
626+
└── filters [type=bool, outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ]), fd=(1)==(2), (2)==(1)]
639627
└── eq [type=bool, outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ])]
640628
├── variable: uv.u [type=int, outer=(2)]
641629
└── variable: column1 [type=int, outer=(1)]
@@ -661,12 +649,10 @@ anti-join-apply
661649
│ ├── outer: (1)
662650
│ ├── cardinality: [0 - 5]
663651
│ ├── stats: [rows=5]
664-
│ ├── fd: (1)==(5), (5)==(1) [removed: (1)]
665652
│ ├── select
666653
│ │ ├── columns: u:5(int!null) v:6(int!null)
667654
│ │ ├── outer: (1)
668655
│ │ ├── stats: [rows=111.111111]
669-
│ │ ├── fd: (1)==(5), (5)==(1) [removed: (1)]
670656
│ │ ├── scan uv
671657
│ │ │ ├── columns: u:5(int) v:6(int!null)
672658
│ │ │ └── stats: [rows=1000]
@@ -979,7 +965,7 @@ right-join
979965
├── columns: u:1(int) sum:4(decimal) x:5(int!null) y:6(int) s:7(string) d:8(decimal!null)
980966
├── stats: [rows=70000]
981967
├── key: (1,5)
982-
├── fd: (1)~~>(4), (5)-->(6-8), (7,8)~~>(5,6), (1,5)-->(4)
968+
├── fd: (5)-->(6-8), (7,8)~~>(5,6), (1)~~>(4), (1,5)-->(4)
983969
├── group-by
984970
│ ├── columns: u:1(int) sum:4(decimal)
985971
│ ├── grouping columns: u:1(int)
@@ -1094,7 +1080,7 @@ full-join
10941080
├── columns: u:1(int) sum:4(decimal) x:5(int) y:6(int) s:7(string) d:8(decimal)
10951081
├── stats: [rows=70000]
10961082
├── key: (1,5)
1097-
├── fd: (1)~~>(4), (5)-->(6-8), (7,8)~~>(5,6), (1,5)-->(4)
1083+
├── fd: (5)-->(6-8), (7,8)~~>(5,6), (1)~~>(4), (1,5)-->(4)
10981084
├── group-by
10991085
│ ├── columns: u:1(int) sum:4(decimal)
11001086
│ ├── grouping columns: u:1(int)

0 commit comments

Comments
 (0)