Skip to content

Commit 9cff1dd

Browse files
jyotiskamateiz
authored andcommitted
Added doctest and method description in context.py
Added doctest for method textFile and description for methods _initialize_context and _ensure_initialized in context.py Author: Jyotiska NK <jyotiska123@gmail.com> Closes alteryx#187 from jyotiska/pyspark_context and squashes the following commits: 356f945 [Jyotiska NK] Added doctest for textFile method in context.py 5b23686 [Jyotiska NK] Updated context.py with method descriptions
1 parent 4dbb27b commit 9cff1dd

File tree

1 file changed

+14
-1
lines changed

1 file changed

+14
-1
lines changed

python/pyspark/context.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,12 +173,18 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
173173
self._temp_dir = \
174174
self._jvm.org.apache.spark.util.Utils.createTempDir(local_dir).getAbsolutePath()
175175

176-
# Initialize SparkContext in function to allow subclass specific initialization
177176
def _initialize_context(self, jconf):
177+
"""
178+
Initialize SparkContext in function to allow subclass specific initialization
179+
"""
178180
return self._jvm.JavaSparkContext(jconf)
179181

180182
@classmethod
181183
def _ensure_initialized(cls, instance=None, gateway=None):
184+
"""
185+
Checks whether a SparkContext is initialized or not.
186+
Throws error if a SparkContext is already running.
187+
"""
182188
with SparkContext._lock:
183189
if not SparkContext._gateway:
184190
SparkContext._gateway = gateway or launch_gateway()
@@ -270,6 +276,13 @@ def textFile(self, name, minPartitions=None):
270276
Read a text file from HDFS, a local file system (available on all
271277
nodes), or any Hadoop-supported file system URI, and return it as an
272278
RDD of Strings.
279+
280+
>>> path = os.path.join(tempdir, "sample-text.txt")
281+
>>> with open(path, "w") as testFile:
282+
... testFile.write("Hello world!")
283+
>>> textFile = sc.textFile(path)
284+
>>> textFile.collect()
285+
[u'Hello world!']
273286
"""
274287
minPartitions = minPartitions or min(self.defaultParallelism, 2)
275288
return RDD(self._jsc.textFile(name, minPartitions), self,

0 commit comments

Comments
 (0)