diff --git a/docs/source/conf.py b/docs/source/conf.py index 1e03009..fe9b85d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,191 +1,31 @@ -# -*- coding: utf-8 -*- -# # Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- project = "pylivy" -copyright = "2018, Andrew Crozier" +copyright = "2018-2020, Andrew Crozier" author = "Andrew Crozier" -# The short X.Y version -version = "" -# The full version, including alpha/beta/rc tags -release = "" - # -- General configuration --------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx_autodoc_typehints", ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" - -# The master toctree document. -master_doc = "index" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - # Order class method documentation according to their order in source code autodoc_member_order = "bysource" # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# html_theme = "alabaster" -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = { "description": "Use Apache Livy from Python.", "github_user": "acroz", "github_repo": "pylivy", } - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = "pylivydoc" - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "pylivy.tex", - "pylivy Documentation", - "Andrew Crozier", - "manual", - ) -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "pylivy", "pylivy Documentation", [author], 1)] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "pylivy", - "pylivy Documentation", - author, - "pylivy", - "Use Apache Livy from Python.", - "Miscellaneous", - ) -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ["search.html"] - - -# -- Extension configuration ------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 9be9497..31206a4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,40 +23,28 @@ Installation Note that ``pylivy`` requires Python 3.6 or later. -Usage ------ +Basic Usage +----------- The :class:`~livy.session.LivySession` class is the main interface provided by ``pylivy``:: from livy import LivySession - LIVY_URL = 'http://spark.example.com:8998' + LIVY_URL = "http://spark.example.com:8998" with LivySession.create(LIVY_URL) as session: # Run some code on the remote cluster session.run("filtered = df.filter(df.name == 'Bob')") # Retrieve the result - local_df = session.read('filtered') - -Authenticate requests sent to Livy by passing `any requests Auth object -`_ to the -``LivySession``. For example, to perform HTTP basic auth do:: - - from requests.auth import HTTPBasicAuth - - auth = HTTPBasicAuth('username', 'password') - - with LivySession.create(LIVY_URL, auth) as session: - session.run("filtered = df.filter(df.name == 'Bob')") - local_df = session.read('filtered') + local_df = session.read("filtered") Similarly, batch sessions in Livy can be created and managed with the :class:`~livy.batch.LivyBatch` class:: from livy import LivyBatch - LIVY_URL = 'http://spark.example.com:8998' + LIVY_URL = "http://spark.example.com:8998" batch = LivyBatch.create( integration_url, @@ -69,6 +57,52 @@ Similarly, batch sessions in Livy can be created and managed with the ) batch.wait() +See :class:`LivySession.create ` or +:class:`LivyBatch.create ` for the full range of +options that can be specified when creating sessions or batches. + +Authentication +-------------- + +Authenticate requests sent to Livy by passing `any requests Auth object +`_ to the +:class:`~livy.session.LivySession`. For example, to perform HTTP basic auth +do:: + + from livy import LivySession + from requests.auth import HTTPBasicAuth + + auth = HTTPBasicAuth("username", "password") + + with LivySession.create(LIVY_URL, auth) as session: + session.run("filtered = df.filter(df.name == 'Bob')") + local_df = session.read("filtered") + +Custom requests session +----------------------- + +``pylivy`` uses `requests `_ to make HTTP +requests to your Livy server. You can specify your own requests session in +order to customise how requests are made to the server. + +For example, to add a custom header to all requests make to Livy:: + + from livy import LivySession + import requests + + LIVY_URL = "http://spark.example.com:8998" + + requests_session = requests.Session() + requests_session.headers.update( + {"X-Auth-Token": "MY-SECURITY-TOKEN"} + ) + + with LivySession.create( + LIVY_URL, + requests_session=requests_session + ) as session: + session.run("filtered = df.filter(df.name == 'Bob')") + API Documentation -----------------