Mergemaster/cloudml (#134)

* Add gcs_copy_file() that is missing but is referenced in a couple of places. (#110)

* Add gcs_copy_file() that is missing but is referenced in a couple of places.

* Add DataFlow to pydatalab dependency list.

* Fix travis test errors by reimplementing gcs copy.

* Remove unnecessary shutil import.

* Flake8 configuration. Set max line length to 100. Ignore E111, E114 (#102)

* Add datalab user agent to CloudML trainer and predictor requests. (#112)

* Update oauth2client to 2.2.0 to satisfy cloudml in Cloud Datalab (#111)

* Update README.md (#114)

Added docs link.

* Generate reST documentation for magic commands (#113)

Auto generate docs for any added magics by searching through the source files for lines with register_line_cell_magic, capturing the names for those magics, and calling them inside an ipython kernel with the -h argument, then storing that output into a generated datalab.magics.rst file.

* Fix an issue that %%chart failed with UDF query. (#116)

* Fix an issue that %%chart failed with UDF query.

The problem is that the query is submitted to BQ without replacing variable values from user namespace.

* Fix chart tests by adding ip.user_ns mock.

* Fix charting test.

* Add missing import "mock".

* Fix chart tests.

* Fix "%%bigquery schema" issue --  the command generates nothing in output. (#119)

* Add some missing dependencies, remove some unused ones (#122)

* Remove scikit-learn and scipy as dependencies
* add more required packages
* Add psutil as dependency
* Update packages versions

* Cleanup (#123)

* Remove unnecessary semicolons

* remove unused imports

* remove unncessary defined variable

* Fix query_metadata tests (#128)

Fix query_metadata tests

* Make the library pip-installable (#125)

This PR adds tensorflow and cloudml in setup.py to make the lib pip-installable. I had to install them explicitly using pip from inside the setup.py script, even though it's not a clean way to do it, it gets around the two issues we have at the moment with these two packags:
- Pypi has Tensorflow version 0.12, while we need 0.11 for the current version of pydatalab. According to the Cloud ML docs, that version exists as a pip package for three supported platforms.
- Cloud ML SDK exists as a pip package, but also not on Pypi, and while we could add it as a dependency link, there exists another package on Pypi called cloudml, and pip ends up installing that instead (see #124). I cannot find a way to force pip to install the package from the link I included.

* Set command description so it is displayed in --help. argparser's format_help() prints description but not help. (#131)

Author: qimingj

.gitignore

@@ -7,3 +7,4 @@ MANIFEST
 build
 .coverage
 dist
+datalab.magics.rst

.travis.yml

@@ -1,12 +1,14 @@
 language: python
 python:
-  - "2.7"
-  - "3.4"
+  - 2.7
+  - 3.5
 
 before_install:
+  - sudo apt-get install -y python-setuptools
   - npm install -g typescript
   - tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts
   - pip install -U pip
+  - pip install -U setuptools
   - pip install .
 
 script: python ./tests/main.py

README.md

@@ -60,3 +60,5 @@ You will also need to set the project ID to use; either set a `PROJECT_ID`
 environment variable to the project name, or call `set_datalab_project_id(name)`
 from within your notebook.
 
+## Documentation
+You can read the Sphinx generated docs at: [http://googledatalab.github.io/pydatalab/](http://googledatalab.github.io/pydatalab/)

datalab/bigquery/commands/_bigquery.py

@@ -1009,6 +1009,14 @@ def _repr_html_table_schema(schema):
   _HTML_TEMPLATE = """
     <div class="bqsv" id="%s"></div>
     <script>
+      require.config({
+        map: {
+          '*': {
+            datalab: 'nbextensions/gcpdatalab'
+          }
+        },
+      });
+
       require(['datalab/bigquery', 'datalab/element!%s',
           'datalab/style!/nbextensions/gcpdatalab/bigquery.css'],
         function(bq, dom) {

datalab/mlalpha/_cloud_predictor.py

@@ -81,6 +81,7 @@ def predict(self, data):
 
     request = self._api.projects().predict(body={'instances': data},
                                            name=self._full_version_name)
+    request.headers['user-agent'] = 'GoogleCloudDataLab/1.0'
     result = request.execute()
     if 'predictions' not in result:
       raise Exception('Invalid response from service. Cannot find "predictions" in response.')

datalab/mlalpha/_cloud_runner.py

@@ -86,4 +86,5 @@ def run(self, job_id=None):
                               discoveryServiceUrl=_CLOUDML_DISCOVERY_URL)
     request = cloudml.projects().jobs().create(body=job,
                                                parent='projects/' + context.project_id)
+    request.headers['user-agent'] = 'GoogleCloudDataLab/1.0'
     return request.execute()

datalab/mlalpha/_dataset.py

@@ -293,7 +293,7 @@ def _scatter3d_plot(self, names, x, y, z, color):
     iplot(fig)
 
   def _plot_x(self, names, x):
-    self._histogram(names, x);
+    self._histogram(names, x)
     if x != self._target_name:
       self._scatter_plot(names, x, self._target_name, self._target_name)
 

datalab/mlalpha/commands/_mlalpha.py

@@ -18,14 +18,13 @@
 
 
 import collections
-import datetime
 import fnmatch
 import google.cloud.ml
 import json
 import math
 import os
 import plotly.graph_objs as go
-from plotly.offline import init_notebook_mode, iplot
+from plotly.offline import iplot
 import urllib
 import yaml
 
@@ -271,7 +270,7 @@ def _train(args, cell):
         urllib.urlencode(log_url_query_strings)
     html += '<p>Click <a href="%s" target="_blank">here</a> to view cloud log. <br/>' % log_url
     html += 'Start TensorBoard by running "%tensorboard start --logdir=&lt;YourLogDir&gt;".</p>'
-    return IPython.core.display.HTML(html);
+    return IPython.core.display.HTML(html)
   else:
     # local training
     package_path = None

datalab/stackdriver/monitoring/_query_metadata.py

@@ -64,7 +64,7 @@ def as_dataframe(self, max_rows=None):
       """
       max_rows = len(self._timeseries_list) if max_rows is None else max_rows
       headers = [{
-          'resource': ts.resource.__dict__, 'metric': ts.metric.__dict__}
+          'resource': ts.resource._asdict(), 'metric': ts.metric._asdict()}
           for ts in self._timeseries_list[:max_rows]]
 
       if not headers:

datalab/storage/_item.py

@@ -196,7 +196,6 @@ def read_lines(self, max_lines=None):
 
     max_to_read = self.metadata.size
     bytes_to_read = min(100 * max_lines, self.metadata.size)
-    lines = []
     while True:
       content = self.read_from(byte_count=bytes_to_read)
 

datalab/utils/__init__.py

@@ -21,4 +21,4 @@
 from ._lru_cache import LRUCache
 from ._lambda_job import LambdaJob
 from ._utils import print_exception_with_last_stack, get_item, compare_datetimes, \
-                    pick_unused_port, is_http_running_on
+                    pick_unused_port, is_http_running_on, gcs_copy_file

datalab/utils/_utils.py

@@ -22,6 +22,7 @@
     import httplib
 
 import pytz
+import subprocess
 import socket
 import traceback
 import types
@@ -110,3 +111,13 @@ def is_http_running_on(port):
     return True
   except Exception as e:
     return False
+
+
+def gcs_copy_file(source, dest):
+  """ Copy file from source to destination. The paths can be GCS or local.
+
+  Args:
+    source: the source file path.
+    dest: the destination file path.
+  """
+  subprocess.check_call(['gsutil', '-q', 'cp', source, dest])

datalab/utils/commands/_commands.py

@@ -79,4 +79,4 @@ def subcommand(self, name, help):
     """Creates a parser for a sub-command. """
     if self._subcommands is None:
       self._subcommands = self.add_subparsers(help='commands')
-    return self._subcommands.add_parser(name, help=help)
+    return self._subcommands.add_parser(name, description=help, help=help)

datalab/utils/commands/_utils.py

@@ -207,10 +207,11 @@ def get_data(source, fields='*', env=None, first_row=0, count=-1, schema=None):
     Exception if the request could not be fulfilled.
   """
 
+  ipy = IPython.get_ipython()
   if env is None:
     env = {}
+  env.update(ipy.user_ns)
   if isinstance(source, basestring):
-    ipy = IPython.get_ipython()
     source = datalab.utils.get_item(ipy.user_ns, source, source)
     if isinstance(source, basestring):
       source = datalab.bigquery.Table(source)

docs/Makefile

@@ -55,38 +55,42 @@ help:
 clean:
 	rm -rf $(BUILDDIR)/*
 
-html:
+pre-build:
+	@echo "Generate reST for magic commands:"
+	ipython gen-magic-rst.ipy
+
+html: pre-build
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: pre-build
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: pre-build
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: pre-build
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: pre-build
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: pre-build
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: pre-build
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -95,15 +99,15 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/api.qhc"
 
-applehelp:
+applehelp: pre-build
 	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
 	@echo
 	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
 	@echo "N.B. You won't be able to view it unless you put it in" \
 	      "~/Library/Documentation/Help or install it in your application" \
 	      "bundle."
 
-devhelp:
+devhelp: pre-build
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -112,85 +116,85 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/api"
 	@echo "# devhelp"
 
-epub:
+epub: pre-build
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-latexpdfja:
+latexpdfja: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through platex and dvipdfmx..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: pre-build
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: pre-build
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-texinfo:
+texinfo: pre-build
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo
 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."
 
-info:
+info: pre-build
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
 
-gettext:
+gettext: pre-build
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
-changes:
+changes: pre-build
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: pre-build
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-doctest:
+doctest: pre-build
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
-coverage:
+coverage: pre-build
 	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
 	@echo "Testing of coverage in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/coverage/python.txt."
 
-xml:
+xml: pre-build
 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
 	@echo
 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
 
-pseudoxml:
+pseudoxml: pre-build
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
@@ -202,7 +206,7 @@ prepublish:
 	cd ../../datalab-docs && git clone https://github.com/GoogleCloudPlatform/datalab.git html && \
 		git checkout gh-pages
 
-publish:
+publish: pre-build
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	cd ../../datalab-docs/html && git add . && git commit -m "Updated" && git push --force origin gh-pages
 

docs/README

@@ -1,9 +1,8 @@
-To use, install the prerequisites:
+To use, install the prerequisites and the pydatalab module:
 
   pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon
+  pip install .. # from docs directory
 
+then in the docs directory, do 'make html' (or epub, or text, etc).
 
-then in the docs directory, do 'make html' (or epub, or pdf, etc).
-
-Output will be in the docs/_build directory.
-
+Output will be in $BUILDDIR, defaulting to ../../datalab-docs.

docs/conf.py

@@ -145,7 +145,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = []
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/gen-magic-rst.ipy

@@ -0,0 +1,39 @@
+import subprocess, pkgutil, importlib, sys
+from cStringIO import StringIO
+
+# ignore mlalpha and tensorboard for now because of their tensorflow dependency
+# until tensorboard is pip installable and can be listed as a pydatalab dependency
+IGNORED_MAGICS = ['mlalpha', 'tensorboard']
+
+# import submodules
+submodules = [s for _,s,_ in pkgutil.iter_modules(['../datalab'])]
+
+for m in submodules:
+  name = 'datalab.' + m + '.commands'
+  try:
+    importlib.import_module(name)
+  except:
+    sys.stderr.write('WARNING, could not find module ' + name + '. Ignoring..\n')
+
+magic_regex = "find ../datalab -name '*.py' -exec perl -e '$f=join(\"\",<>); print \"$1\n\" if $f=~/register_line_cell_magic\ndef ([^\(]+)/m' {} \;"
+magics = subprocess.check_output(magic_regex, shell=True)
+
+reSTfile = open('datalab.magics.rst', 'w')
+indent = '\n  '
+
+reSTfile.write('datalab.magics\n')
+reSTfile.write('=================\n\n')
+
+for m in magics.split():
+  if m in IGNORED_MAGICS:
+    sys.stderr.write('Ignoring magic ' + m + '\n')
+  else:
+    reSTfile.write('.. attribute:: ' + m + '\n')
+    reSTfile.write('.. parsed-literal::\n')
+    # hijack stdout since the ipython kernel call writes to stdout/err directly
+    # and does not return its output
+    tmpStdout, sys.stdout = sys.stdout, StringIO()
+    get_ipython().magic(m + ' -h')
+    resultout = sys.stdout.getvalue().splitlines()
+    sys.stdout = tmpStdout
+    reSTfile.writelines(indent + indent.join(resultout) + '\n\n')