File tree Expand file tree Collapse file tree 1 file changed +3
-1
lines changed Expand file tree Collapse file tree 1 file changed +3
-1
lines changed Original file line number Diff line number Diff line change 33
33
34
34
from pyspark .resultiterable import ResultIterable
35
35
36
+
36
37
def _do_python_join (rdd , other , numPartitions , dispatch ):
37
38
vs = rdd .map (lambda (k , v ): (k , (1 , v )))
38
39
ws = other .map (lambda (k , v ): (k , (2 , v )))
39
- return vs .union (ws ).groupByKey (numPartitions ).flatMapValues (lambda x : dispatch (x .__iter__ ()))
40
+ return vs .union (ws ).groupByKey (numPartitions ).flatMapValues (lambda x : dispatch (x .__iter__ ()))
40
41
41
42
42
43
def python_join (rdd , other , numPartitions ):
@@ -85,6 +86,7 @@ def make_mapper(i):
85
86
vrdds = [rdd .map (make_mapper (i )) for i , rdd in enumerate (rdds )]
86
87
union_vrdds = reduce (lambda acc , other : acc .union (other ), vrdds )
87
88
rdd_len = len (vrdds )
89
+
88
90
def dispatch (seq ):
89
91
bufs = [[] for i in range (rdd_len )]
90
92
for (n , v ) in seq :
You can’t perform that action at this time.
0 commit comments