Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

i13 python api #58

Merged
merged 21 commits into from
Jan 25, 2018
Merged

i13 python api #58

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3fb8774
WIP: Initial commit - Python wrappers for VariantSpark core API classes
ppillay Dec 13, 2017
b18a6cb
WIP: Test now works
ppillay Dec 13, 2017
cdd1c10
Missed file
ppillay Dec 13, 2017
02e72b0
Fixed Travis CI feedback.
ppillay Dec 15, 2017
c1faca8
Fixed Travis CI feedback
ppillay Dec 15, 2017
3bf83fb
Updates following review: doc fixes, file renames, new analysis metho…
ppillay Jan 19, 2018
6a8b333
Missed commit of deleted file
ppillay Jan 19, 2018
1e3d901
Re-ordered imports as per pylint
ppillay Jan 19, 2018
7d74c18
Made tests discoverable
ppillay Jan 19, 2018
3f0785b
Removed old test file
ppillay Jan 19, 2018
5996160
Added configuration to make test environment and outcomes consistent
ppillay Jan 22, 2018
c6ce8b4
Ignore pyspark import issues
ppillay Jan 22, 2018
4d92e5a
Backed out pyspark install
ppillay Jan 22, 2018
79fe12f
Merge pull request #54 from ppillay/Issue-13-Python-API
piotrszul Jan 24, 2018
056fb7d
Added classpath to variant-spark jar in text context and removed pysp…
piotrszul Jan 24, 2018
37645fa
Refactored tests to use PROJECT_DIR
piotrszul Jan 24, 2018
20da63a
Added inclusion of jar file to python distribution
piotrszul Jan 24, 2018
51f7d10
Added a simple example and ignore exception when pyspark not present
piotrszul Jan 25, 2018
9e40159
Fixed pylint errors
piotrszul Jan 25, 2018
4ab53bd
For now remove running of the test form the build
piotrszul Jan 25, 2018
abfa45e
Added pyton docs
piotrszul Jan 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixed Travis CI feedback.
  • Loading branch information
@ppillay
ppillay committed Dec 15, 2017
commit 02e72b00dff11d8fab850872d65343058b9bb3dd
2 changes: 1 addition & 1 deletion python/variants/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from variants.core import VariantsContext
from variants.core import VariantsContext
32 changes: 19 additions & 13 deletions python/variants/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class VariantsContext(object):
:type sc: :class:`.pyspark.SparkSession`

"""

def __init__(self, ss=None):
self.sc = ss.sparkContext
self.sql = SQLContext.getOrCreate(self.sc)
Expand All @@ -23,19 +24,19 @@ def __init__(self, ss=None):
if self.sc._jsc.sc().uiWebUrl().isDefined():
sys.stderr.write('SparkUI available at {}\n'.format(self.sc._jsc.sc().uiWebUrl().get()))
sys.stderr.write(
'Welcome to\n'
' _ __ _ __ _____ __ \n'
'| | / /___ ______(_)___ _____ / /_/ ___/____ ____ ______/ /__ \n'
'| | / / __ `/ ___/ / __ `/ __ \/ __/\__ \/ __ \/ __ `/ ___/ //_/ \n'
'| |/ / /_/ / / / / /_/ / / / / /_ ___/ / /_/ / /_/ / / / ,< \n'
'|___/\__,_/_/ /_/\__,_/_/ /_/\__//____/ .___/\__,_/_/ /_/|_| \n'
' /_/ \n')

'Welcome to\n'
' _ __ _ __ _____ __ \n'
'| | / /___ ______(_)___ _____ / /_/ ___/____ ____ ______/ /__ \n'
'| | / / __ `/ ___/ / __ `/ __ \/ __/\__ \/ __ \/ __ `/ ___/ //_/ \n'
'| |/ / /_/ / / / / /_/ / / / / /_ ___/ / /_/ / /_/ / / / ,< \n'
'|___/\__,_/_/ /_/\__,_/_/ /_/\__//____/ .___/\__,_/_/ /_/|_| \n'
' /_/ \n')

def import_vcf(self, vcf_file_path):
""" Shut down the VariantsContext.
"""
return VCFFeatureSource(self._jvm, self._vs_api, self._jsql, self._jvsc.importVCF(vcf_file_path))
return VCFFeatureSource(self._jvm, self._vs_api,
self._jsql, self._jvsc.importVCF(vcf_file_path))

def load_label(self, label_file_path, col_name):
""" Loads the label source file
Expand All @@ -52,6 +53,7 @@ def stop(self):
self.sc.stop()
self.sc = None


class VCFFeatureSource(object):

def __init__(self, _jvm, _vs_api, _jsql, _jvcffs):
Expand All @@ -60,8 +62,8 @@ def __init__(self, _jvm, _vs_api, _jsql, _jvcffs):
self._vs_api = _vs_api
self._jsql = _jsql

def importance_analysis(self, labelSource, n_trees=1000, mtry_fraction=None, oob=True, seed=None,
batch_size=100, varOrdinalLevels=3):
def importance_analysis(self, labelSource, n_trees=1000, mtry_fraction=None,
oob=True, seed=None, batch_size=100, varOrdinalLevels=3):
"""Builds random forest classifier.

:param labelSource: The ingested label source
Expand All @@ -75,13 +77,17 @@ def importance_analysis(self, labelSource, n_trees=1000, mtry_fraction=None, oob
:return: Importance analysis model.
:rtype: :py:class:`ImportanceAnalysis`
"""
rfParams = self._jvm.au.csiro.variantspark.algo.RandomForestParams(bool(oob), float(mtry_fraction), True, float('nan'), True, long(seed))
ia = self._vs_api.ImportanceAnalysis(self._jsql ,self._jvcffs, labelSource, rfParams, n_trees, batch_size, varOrdinalLevels)
rfParams = self._jvm.au.csiro.variantspark.algo.RandomForestParams(bool(oob), float(mtry_fraction),
True, float('nan'), True, long(seed))
ia = self._vs_api.ImportanceAnalysis(self._jsql, self._jvcffs, labelSource,
rfParams, n_trees, batch_size, varOrdinalLevels)
return ImportanceAnalysis(ia)


class ImportanceAnalysis(object):
""" Model for random forest based importance analysis
"""

def __init__(self, _jia):
self._jia = _jia

Expand Down
3 changes: 2 additions & 1 deletion python/variants/tests/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ def test_variants_context(self):
most_imp_var = imp_vars.head()._1()
self.assertEqual(most_imp_var, '22_16050408')


if __name__ == '__main__':
unittest.main(verbosity = 2)
unittest.main(verbosity=2)