ARROW-13729: [Website] setup datafusion python binding docs

.dockerignore

@@ -57,6 +57,4 @@
 !rust/parquet/build.rs
 !rust/parquet_derive/Cargo.toml
 !rust/parquet_derive_test/Cargo.toml
-!rust/datafusion/Cargo.toml
-!rust/datafusion/benches
 !rust/integration-testing/Cargo.toml

docs/.gitignore

@@ -17,3 +17,6 @@
 
 _build
 source/python/generated
+source/datafusion/generated
+venv/
+bigfile.arrow

docs/environment.yml

@@ -16,10 +16,11 @@
 # under the License.
 
 channels:
-- defaults
-- conda-forge
+  - defaults
+  - conda-forge
 dependencies:
-- arrow-cpp
-- parquet-cpp
-- pyarrow
-- numpydoc
+  - arrow-cpp
+  - parquet-cpp
+  - pyarrow
+  - datafusion
+  - numpydoc

docs/source/conf.py

@@ -37,14 +37,10 @@
 import warnings
 from unittest import mock
 
+import datafusion
 import pyarrow
 
-
-sys.path.extend([
-    os.path.join(os.path.dirname(__file__),
-                 '..', '../..')
-
-])
+sys.path.extend([os.path.join(os.path.dirname(__file__), "..", "../..")])
 
 # Suppresses all warnings printed when sphinx is traversing the code (e.g.
 # deprecation warnings)
@@ -60,24 +56,24 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.doctest',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'IPython.sphinxext.ipython_directive',
-    'IPython.sphinxext.ipython_console_highlighting',
-    'breathe'
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.doctest",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
+    "IPython.sphinxext.ipython_directive",
+    "IPython.sphinxext.ipython_console_highlighting",
+    "breathe",
 ]
 
 # Show members for classes in .. autosummary
 autodoc_default_options = {
-    'members': None,
-    'undoc-members': None,
-    'show-inheritance': None,
-    'inherited-members': None
+    "members": None,
+    "undoc-members": None,
+    "show-inheritance": None,
+    "inherited-members": None,
 }
 
 # Breathe configuration
@@ -88,19 +84,19 @@
 autodoc_mock_imports = []
 
 # ipython directive options
-ipython_mplbackend = ''
+ipython_mplbackend = ""
 
 # numpydoc configuration
 napoleon_use_rtype = False
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 
-source_suffix = ['.rst']
+source_suffix = [".rst"]
 
 autosummary_generate = True
 
@@ -109,23 +105,21 @@
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'Apache Arrow'
-copyright = f'2016-{datetime.datetime.now().year} Apache Software Foundation'
-author = u'Apache Software Foundation'
+project = "Apache Arrow"
+copyright = f"2016-{datetime.datetime.now().year} Apache Software Foundation"
+author = "Apache Software Foundation"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = os.environ.get('ARROW_DOCS_VERSION',
-                         pyarrow.__version__)
+version = os.environ.get("ARROW_DOCS_VERSION", pyarrow.__version__)
 # The full version, including alpha/beta/rc tags.
-release = os.environ.get('ARROW_DOCS_VERSION',
-                         pyarrow.__version__)
+release = os.environ.get("ARROW_DOCS_VERSION", pyarrow.__version__)
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -146,7 +140,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -168,7 +162,7 @@
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -185,7 +179,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'pydata_sphinx_theme'
+html_theme = "pydata_sphinx_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -202,7 +196,7 @@
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.
 #
-html_title = u'Apache Arrow v{}'.format(version)
+html_title = "Apache Arrow v{}".format(version)
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
 #
@@ -222,10 +216,10 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Custom fixes to the RTD theme
-html_css_files = ['theme_overrides.css']
+html_css_files = ["theme_overrides.css"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -247,8 +241,8 @@
 # Custom sidebar templates, maps document names to template names.
 #
 html_sidebars = {
-#    '**': ['sidebar-logo.html', 'sidebar-search-bs.html', 'sidebar-nav-bs.html'],
-    '**': ['docs-sidebar.html'],
+    #    '**': ['sidebar-logo.html', 'sidebar-search-bs.html', 'sidebar-nav-bs.html'],
+    "**": ["docs-sidebar.html"],
 }
 
 # Additional templates that should be rendered to pages, maps page names to
@@ -308,34 +302,36 @@
 # html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'arrowdoc'
+htmlhelp_basename = "arrowdoc"
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-     # The paper size ('letterpaper' or 'a4paper').
-     #
-     # 'papersize': 'letterpaper',
-
-     # The font size ('10pt', '11pt' or '12pt').
-     #
-     # 'pointsize': '10pt',
-
-     # Additional stuff for the LaTeX preamble.
-     #
-     # 'preamble': '',
-
-     # Latex figure (float) alignment
-     #
-     # 'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'arrow.tex', u'Apache Arrow Documentation',
-     u'Apache Arrow Team', 'manual'),
+    (
+        master_doc,
+        "arrow.tex",
+        "Apache Arrow Documentation",
+        "Apache Arrow Team",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -375,10 +371,7 @@
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'arrow', u'Apache Arrow Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "arrow", "Apache Arrow Documentation", [author], 1)]
 
 # If true, show URL addresses after external links.
 #
@@ -391,9 +384,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'arrow', u'Apache Arrow Documentation',
-     author, 'Apache Arrow', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "arrow",
+        "Apache Arrow Documentation",
+        author,
+        "Apache Arrow",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
@@ -427,24 +426,26 @@
 
 try:
     import pyarrow.cuda
+
     cuda_enabled = True
 except ImportError:
     cuda_enabled = False
     # Mock pyarrow.cuda to avoid autodoc warnings.
     # XXX I can't get autodoc_mock_imports to work, so mock manually instead
     # (https://github.com/sphinx-doc/sphinx/issues/2174#issuecomment-453177550)
-    pyarrow.cuda = sys.modules['pyarrow.cuda'] = mock.Mock()
+    pyarrow.cuda = sys.modules["pyarrow.cuda"] = mock.Mock()
 
 try:
     import pyarrow.flight
+
     flight_enabled = True
 except ImportError:
     flight_enabled = False
-    pyarrow.flight = sys.modules['pyarrow.flight'] = mock.Mock()
+    pyarrow.flight = sys.modules["pyarrow.flight"] = mock.Mock()
 
 
 def setup(app):
     # Use a config value to indicate whether CUDA API docs can be generated.
     # This will also rebuild appropriately when the value changes.
-    app.add_config_value('cuda_enabled', cuda_enabled, 'env')
-    app.add_config_value('flight_enabled', flight_enabled, 'env')
+    app.add_config_value("cuda_enabled", cuda_enabled, "env")
+    app.add_config_value("flight_enabled", flight_enabled, "env")

docs/source/datafusion/api.rst

@@ -0,0 +1,30 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. _api:
+
+*************
+API Reference
+*************
+
+.. toctree::
+   :maxdepth: 2
+
+   api/dataframe
+   api/execution_context
+   api/expression
+   api/functions

docs/source/datafusion/api/dataframe.rst

@@ -0,0 +1,27 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. _api.dataframe:
+.. currentmodule:: datafusion
+
+DataFrame
+=========
+
+.. autosummary::
+   :toctree: ../generated/
+
+   datafusion.DataFrame