Skip to content

Commit ccfd214

Browse files
Ken Takagiwagiwa
Ken Takagiwa
authored and
giwa
committed
added doctest for pyspark.streaming.duration
1 parent 0d1b954 commit ccfd214

File tree

3 files changed

+233
-30
lines changed

3 files changed

+233
-30
lines changed

‎python/pyspark/streaming/duration.py

Lines changed: 214 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -42,65 +42,164 @@ def __init__(self, millis, _jvm=None):
4242
self._jduration = _jvm.Duration(millis)
4343

4444
def toString(self):
45-
""" Return duration as string """
45+
"""
46+
Return duration as string
47+
48+
>>> d_10 = Duration(10)
49+
>>> d_10.toString()
50+
'10 ms'
51+
"""
4652
return str(self._millis) + " ms"
4753

4854
def isZero(self):
49-
""" Check if millis is zero """
55+
"""
56+
Check if millis is zero
57+
58+
>>> d_10 = Duration(10)
59+
>>> d_10.isZero()
60+
False
61+
>>> d_0 = Duration(0)
62+
>>> d_0.isZero()
63+
True
64+
"""
5065
return self._millis == 0
5166

5267
def prettyPrint(self):
5368
"""
5469
Return a human-readable string representing a duration
70+
71+
>>> d_10 = Duration(10)
72+
>>> d_10.prettyPrint()
73+
'10 ms'
74+
>>> d_1sec = Duration(1000)
75+
>>> d_1sec.prettyPrint()
76+
'1.0 s'
77+
>>> d_1min = Duration(60 * 1000)
78+
>>> d_1min.prettyPrint()
79+
'1.0 m'
80+
>>> d_1hour = Duration(60 * 60 * 1000)
81+
>>> d_1hour.prettyPrint()
82+
'1.00 h'
5583
"""
5684
return utils.msDurationToString(self._millis)
5785

5886
def milliseconds(self):
59-
""" Return millisecond """
87+
"""
88+
Return millisecond
89+
90+
>>> d_10 = Duration(10)
91+
>>> d_10.milliseconds()
92+
10
93+
94+
"""
6095
return self._millis
6196

6297
def toFormattedString(self):
63-
""" Return millisecond """
98+
"""
99+
Return millisecond
100+
101+
>>> d_10 = Duration(10)
102+
>>> d_10.toFormattedString()
103+
'10'
104+
105+
"""
64106
return str(self._millis)
65107

66108
def max(self, other):
67-
""" Return higher Duration """
109+
"""
110+
Return higher Duration
111+
112+
>>> d_10 = Duration(10)
113+
>>> d_100 = Duration(100)
114+
>>> d_max = d_10.max(d_100)
115+
>>> print d_max
116+
100 ms
117+
118+
"""
68119
Duration._is_duration(other)
69120
if self > other:
70121
return self
71122
else:
72123
return other
73124

74125
def min(self, other):
75-
""" Return lower Durattion """
126+
"""
127+
Return lower Durattion
128+
129+
>>> d_10 = Duration(10)
130+
>>> d_100 = Duration(100)
131+
>>> d_min = d_10.min(d_100)
132+
>>> print d_min
133+
10 ms
134+
135+
"""
76136
Duration._is_duration(other)
77137
if self < other:
78138
return self
79139
else:
80140
return other
81141

82142
def __str__(self):
143+
"""
144+
>>> d_10 = Duration(10)
145+
>>> str(d_10)
146+
'10 ms'
147+
148+
"""
83149
return self.toString()
84150

85151
def __add__(self, other):
86-
""" Add Duration and Duration """
152+
"""
153+
Add Duration and Duration
154+
155+
>>> d_10 = Duration(10)
156+
>>> d_100 = Duration(100)
157+
>>> d_110 = d_10 + d_100
158+
>>> print d_110
159+
110 ms
160+
"""
87161
Duration._is_duration(other)
88162
return Duration(self._millis + other._millis)
89163

90164
def __sub__(self, other):
91-
""" Subtract Duration by Duration """
165+
"""
166+
Subtract Duration by Duration
167+
168+
>>> d_10 = Duration(10)
169+
>>> d_100 = Duration(100)
170+
>>> d_90 = d_100 - d_10
171+
>>> print d_90
172+
90 ms
173+
174+
"""
92175
Duration._is_duration(other)
93176
return Duration(self._millis - other._millis)
94177

95178
def __mul__(self, other):
96-
""" Multiple Duration by Duration """
179+
"""
180+
Multiple Duration by Duration
181+
182+
>>> d_10 = Duration(10)
183+
>>> d_100 = Duration(100)
184+
>>> d_1000 = d_10 * d_100
185+
>>> print d_1000
186+
1000 ms
187+
188+
"""
97189
Duration._is_duration(other)
98190
return Duration(self._millis * other._millis)
99191

100192
def __div__(self, other):
101193
"""
102194
Divide Duration by Duration
103195
for Python 2.X
196+
197+
>>> d_10 = Duration(10)
198+
>>> d_20 = Duration(20)
199+
>>> d_2 = d_20 / d_10
200+
>>> print d_2
201+
2 ms
202+
104203
"""
105204
Duration._is_duration(other)
106205
return Duration(self._millis / other._millis)
@@ -109,46 +208,121 @@ def __truediv__(self, other):
109208
"""
110209
Divide Duration by Duration
111210
for Python 3.0
211+
212+
>>> d_10 = Duration(10)
213+
>>> d_20 = Duration(20)
214+
>>> d_2 = d_20 / d_10
215+
>>> print d_2
216+
2 ms
217+
112218
"""
113219
Duration._is_duration(other)
114220
return Duration(self._millis / other._millis)
115221

116222
def __floordiv__(self, other):
117-
""" Divide Duration by Duration """
223+
"""
224+
Divide Duration by Duration
225+
226+
>>> d_10 = Duration(10)
227+
>>> d_3 = Duration(3)
228+
>>> d_3 = d_10 // d_3
229+
>>> print d_3
230+
3 ms
231+
232+
"""
118233
Duration._is_duration(other)
119234
return Duration(self._millis // other._millis)
120235

121-
def __len__(self):
122-
""" Length of miilisecond in Duration """
123-
return len(self._millis)
124-
125236
def __lt__(self, other):
126-
""" Duration < Duration """
237+
"""
238+
Duration < Duration
239+
240+
>>> d_10 = Duration(10)
241+
>>> d_20 = Duration(20)
242+
>>> d_10 < d_20
243+
True
244+
>>> d_20 < d_10
245+
False
246+
247+
"""
127248
Duration._is_duration(other)
128249
return self._millis < other._millis
129250

130251
def __le__(self, other):
131-
""" Duration <= Duration """
252+
"""
253+
Duration <= Duration
254+
255+
>>> d_10 = Duration(10)
256+
>>> d_20 = Duration(20)
257+
>>> d_10 <= d_20
258+
True
259+
>>> d_20 <= d_10
260+
False
261+
262+
"""
132263
Duration._is_duration(other)
133-
return self.millis <= other._millis
264+
return self._millis <= other._millis
134265

135266
def __eq__(self, other):
136-
""" Duration == Duration """
267+
"""
268+
Duration == Duration
269+
270+
>>> d_10 = Duration(10)
271+
>>> d_20 = Duration(20)
272+
>>> d_10 == d_20
273+
False
274+
>>> other_d_10 = Duration(10)
275+
>>> d_10 == other_d_10
276+
True
277+
278+
"""
137279
Duration._is_duration(other)
138280
return self._millis == other._millis
139281

140282
def __ne__(self, other):
141-
""" Duration != Duration """
283+
"""
284+
Duration != Duration
285+
286+
>>> d_10 = Duration(10)
287+
>>> d_20 = Duration(20)
288+
>>> d_10 != d_20
289+
True
290+
>>> other_d_10 = Duration(10)
291+
>>> d_10 != other_d_10
292+
False
293+
294+
"""
142295
Duration._is_duration(other)
143296
return self._millis != other._millis
144297

145298
def __gt__(self, other):
146-
""" Duration > Duration """
299+
"""
300+
Duration > Duration
301+
302+
>>> d_10 = Duration(10)
303+
>>> d_20 = Duration(20)
304+
>>> d_10 > d_20
305+
False
306+
>>> d_20 > d_10
307+
True
308+
309+
"""
147310
Duration._is_duration(other)
148311
return self._millis > other._millis
149312

150313
def __ge__(self, other):
151-
""" Duration >= Duration """
314+
"""
315+
Duration >= Duration
316+
317+
>>> d_10 = Duration(10)
318+
>>> d_20 = Duration(20)
319+
>>> d_10 < d_20
320+
True
321+
>>> d_20 < d_10
322+
False
323+
324+
325+
"""
152326
Duration._is_duration(other)
153327
return self._millis >= other._millis
154328

@@ -162,25 +336,37 @@ def Milliseconds(milliseconds):
162336
"""
163337
Helper function that creates instance of [[pysparkstreaming.duration]] representing
164338
a given number of milliseconds.
339+
340+
>>> milliseconds = Milliseconds(1)
341+
>>> d_1 = Duration(1)
342+
>>> milliseconds == d_1
343+
True
344+
165345
"""
166346
return Duration(milliseconds)
167347

168348
def Seconds(seconds):
169349
"""
170350
Helper function that creates instance of [[pysparkstreaming.duration]] representing
171351
a given number of seconds.
352+
353+
>>> seconds = Seconds(1)
354+
>>> d_1sec = Duration(1000)
355+
>>> seconds == d_1sec
356+
True
357+
172358
"""
173359
return Duration(seconds * 1000)
174360

175-
def Minites(minites):
361+
def Minutes(minutes):
176362
"""
177363
Helper function that creates instance of [[pysparkstreaming.duration]] representing
178364
a given number of minutes.
179-
"""
180-
return Duration(minutes * 60000)
181365
182-
if __name__ == "__main__":
183-
d = Duration(1)
184-
print d
185-
print d.milliseconds()
366+
>>> minutes = Minutes(1)
367+
>>> d_1min = Duration(60 * 1000)
368+
>>> minutes == d_1min
369+
True
186370
371+
"""
372+
return Duration(minutes * 60 * 1000)

‎python/pyspark/streaming/utils.py

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
1-
__author__ = 'ktakagiw'
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
218

319
def msDurationToString(ms):
420
"""
@@ -12,7 +28,7 @@ def msDurationToString(ms):
1228
return "%d ms" % ms
1329
elif ms < minute:
1430
return "%.1f s" % (float(ms) / second)
15-
elif ms < hout:
31+
elif ms < hour:
1632
return "%.1f m" % (float(ms) / minute)
1733
else:
1834
return "%.2f h" % (float(ms) / hour)

‎python/run-tests

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ export PYSPARK_DOC_TEST=1
7070
run_test "pyspark/broadcast.py"
7171
run_test "pyspark/accumulators.py"
7272
run_test "pyspark/serializers.py"
73+
run_test "pyspark/streaming/duration.py"
7374
unset PYSPARK_DOC_TEST
7475
run_test "pyspark/shuffle.py"
7576
run_test "pyspark/tests.py"

0 commit comments

Comments
 (0)