build(api): Add shared-data to wheel

And do it in a way that doesn’t require copying it into the source tree.

By overriding the build_py subcommand of distutils, which is a core element of
basically all the following distutils commands like sdist or install or (most
crucially) bdist_wheel, we can insert our shared-data into the build directory
which means it will be present in all the following artifacts.

Closes #2452

api/.coveragerc

@@ -1,9 +1,8 @@
 [run]
-omit =
-    */python?.?/*
-    */lib-python/?.?/*.py
-    */lib_pypy/_*.py
-    */site-packages/ordereddict.py
-    */site-packages/nose/*
-    */tests/*
-    opentrons/_version.py
+include =
+    */site-packages/opentrons/*
+
+[paths]
+source =
+    src/opentrons
+    */site-packages/opentrons

api/MANIFEST.in

@@ -1,8 +1,3 @@
-include opentrons/package.json
-include opentrons/config/containers/default-containers.json
-include opentrons/config/smoothie/smoothie-defaults.ini
-include opentrons/config/smoothie/config_one_pro_plus
-include opentrons/config/modules/avrdude.conf
-include opentrons/config/modules/95-opentrons-modules.rules
-include opentrons/config/pipette-config.json
-recursive-include opentrons/resources *
+include src/opentrons/package.json
+graft src/opentrons/config
+graft src/opentrons/resources

api/Makefile

@@ -11,13 +11,21 @@ firmware = $(wildcard smoothie/*.hex)
 
 # python and pipenv config
 python := pipenv run python
+pip := pipenv run pip
+pytest := pipenv run py.test
 pipenv_opts := --dev
 pipenv_opts += $(and $(CI),--ignore-pipfile)
 
+tests ?= tests
+test_opts ?=
+
 .PHONY: install
 install:
 	pipenv install $(pipenv_opts)
 
+.PHONY: all
+all: lint test
+
 .PHONY: clean
 clean:
 	shx rm -rf \
@@ -30,17 +38,13 @@ clean:
 		'**/*.pyc'
 
 .PHONY: test
-test:
-	$(python) -m pytest \
-		--cov=opentrons \
-		--cov-report term-missing:skip-covered \
-		--cov-report xml:coverage.xml \
-		tests
+test: local-install
+	${pytest} ${tests} ${test_opts}
 
 .PHONY: lint
 lint:
-	$(python) -m mypy opentrons
-	$(python) -m pylama opentrons tests
+	$(python) -m mypy src/opentrons
+	$(python) -m pylama src/opentrons tests
 
 .PHONY: docs
 docs:
@@ -55,15 +59,23 @@ publish:
 
 .PHONY: dev
 dev: export ENABLE_VIRTUAL_SMOOTHIE := true
-dev:
-	$(python) opentrons/main.py -P 31950
+dev: local-install
+	$(python) -m opentrons.main -P 31950
 
 .PHONY: wheel
 wheel: clean
 	$(python) setup.py bdist_wheel
 	shx rm -rf build
 	shx ls dist
 
+.PHONY: local-install
+local-install: wheel
+	$(pip) install --ignore-installed --no-deps dist/opentrons-*.whl
+
+.PHONY: local-shell
+local-shell: local-install
+	pipenv shell
+
 .PHONY: push
 push: wheel
 	curl -X POST \

api/README.rst

@@ -102,3 +102,10 @@ Below is a short protocol that will pick up a tip and use it to move 100ul volum
       p300.dispense(plate[i + 1])
 
   p300.return_tip()
+
+Using This Repo Outside Of A Robot
+----------------------------------
+
+The code in this subdirectory can be used outside of a robot to check protocols; however, because the code requires extra shared data files and dependencies, you cannot simply run a python interpreter.
+
+To use the opentrons API locally, please run ``make install`` to set up your system. Then, running ``make local-shell`` will give you a shell with the opentrons module all set up and ready to be imported.

api/codecov.yml

@@ -0,0 +1,2 @@
+fixes:
+  -"src/opentrons/"::"opentrons/"

api/pylama.ini

@@ -5,30 +5,30 @@ linters = mccabe,pep8,pyflakes
 [pylama:mccabe]
 complexity = 9
 
-[pylama:opentrons/server/tests/data/bad_protocol.py]
+[pylama:src/opentrons/server/tests/data/bad_protocol.py]
 skip = 1
 
 # For Windows
-[pylama:opentrons\server\tests\data\bad_protocol.py]
+[pylama:src\opentrons\server\tests\data\bad_protocol.py]
 skip = 1
 
-[pylama:opentrons/server/tests/data/empty.py]
+[pylama:src/opentrons/server/tests/data/empty.py]
 skip = 1
 
 # For Windows
-[pylama:opentrons\server\tests\data\empty.py]
+[pylama:src\opentrons\server\tests\data\empty.py]
 skip = 1
 
-[pylama:opentrons/_version.py]
+[pylama:src/opentrons/_version.py]
 skip = 1
 
 # For Windows
-[pylama:opentrons\_version.py]
+[pylama:src\opentrons\_version.py]
 skip = 1
 
-[pylama:opentrons/resources/jupyter/jupyter_notebook_config.py]
+[pylama:src/opentrons/resources/jupyter/jupyter_notebook_config.py]
 skip = 1
 
 # For Windows
-[pylama:opentrons\resources\jupyter\jupyter_notebook_config.py]
+[pylama:src\opentrons\resources\jupyter\jupyter_notebook_config.py]
 skip = 1

api/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --cov --cov-report term-missing:skip-covered --cov-report xml:coverage.xml

api/setup.py

@@ -2,15 +2,80 @@
 # https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/
 import codecs
 import os
-import shutil
+import os.path
 from setuptools import setup, find_packages
+from setuptools.command import build_py, sdist
+
 import json
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
+# Where we get our files from
+SHARED_DATA_PATH = os.path.join('..', 'shared-data')
+# The subdirectories of SHARED_DATA_PATH to scan for files
+SHARED_DATA_SUBDIRS = ['labware-json-schema',
+                       'protocol-json-schema',
+                       'definitions',
+                       'robot-data']
+# Where, relative to the package root, we put the files we copy
+DEST_BASE_PATH = 'shared_data'
+
+
+def get_shared_data_files():
+    to_include = []
+    for subdir in SHARED_DATA_SUBDIRS:
+        top = os.path.join(SHARED_DATA_PATH, subdir)
+        for dirpath, dirnames, filenames in os.walk(top):
+            from_source = os.path.relpath(dirpath, SHARED_DATA_PATH)
+            to_include.extend([os.path.join(from_source, fname)
+                               for fname in filenames])
+    return to_include
+
+
+class SDistWithSharedData(sdist.sdist):
+    description = sdist.sdist.description\
+        + " Also, include opentrons data files."
+
+    def make_release_tree(self, base_dir, files):
+        self.announce("adding opentrons data files to base dir {}".format(base_dir))
+        for data_file in get_shared_data_files():
+            sdist_dest = os.path.join(base_dir, DEST_BASE_PATH)
+            self.mkpath(os.path.join(sdist_dest, 'opentrons',
+                                     os.path.dirname(data_file)))
+            self.copy_file(os.path.join(SHARED_DATA_PATH, data_file),
+                           os.path.join(sdist_dest, data_file))
+        super().make_release_tree(base_dir, files)
+
+
+class BuildWithSharedData(build_py.build_py):
+    description = build_py.build_py.description\
+        + " Also, include opentrons data files"
+
+    def _get_data_files(self):
+        """
+        Override of build_py.get_data_files that includes out of tree configs.
+        These are currently hardcoded to include everything in
+         ../shared-data/robot-data, which will move to
+        opentrons/config/shared-data
+        """
+        files = super()._get_data_files()
+        # We don’t really want to duplicate logic used in the original
+        # implementation, but we can back out what it did with commonpath -
+        # should be something ending in opentrons
+        build_base = os.path.commonpath([f[2] for f in files])
+        # We want a list of paths to only files relative to ../shared-data
+        to_include = get_shared_data_files()
+        destination = os.path.join(build_base, DEST_BASE_PATH)
+        # And finally, tell the system about our files
+        print("FILES BEFORE {}".format(files))
+        files.append(('opentrons', SHARED_DATA_PATH,
+                      destination, to_include))
+        print("FILES AFTER {}".format(files))
+        return files
+
 
 def get_version():
-    with open(os.path.join(HERE, 'opentrons', 'package.json')) as pkg:
+    with open(os.path.join(HERE, 'src', 'opentrons', 'package.json')) as pkg:
         package_json = json.load(pkg)
         return package_json.get('version')
 
@@ -37,13 +102,14 @@ def get_version():
 DESCRIPTION = (
     "The Opentrons API is a simple framework designed to make "
     "writing automated biology lab protocols easy.")
-PACKAGES = find_packages(where='.', exclude=["tests.*", "tests"])
+PACKAGES = find_packages(where='src')
 INSTALL_REQUIRES = [
     'pyserial==3.2.1',
     'aiohttp==2.3.8',
     'numpy==1.12.1',
     'urwid==1.3.1']
 
+
 def read(*parts):
     """
     Build an absolute path from *parts* and and return the contents of the
@@ -54,20 +120,6 @@ def read(*parts):
 
 
 if __name__ == "__main__":
-    pipette_config_filename = 'pipette-config.json'
-    config_src = os.path.join(
-        '..', 'shared-data', 'robot-data', pipette_config_filename)
-    config_dst = os.path.join('opentrons', 'config')
-    # If you add more copies like this in setup.py you must add them to the
-    # Dockerfile as well, since this doesn’t work during a docker build
-    try:
-        pipette_config_file = os.path.join(config_dst, pipette_config_filename)
-        if os.path.exists(pipette_config_file):
-            os.remove(pipette_config_file)
-        shutil.copy2(config_src, config_dst)
-    except OSError:
-        print('Unable to copy shared data directory due to exception:')
-
     setup(
         python_requires='>=3.6',
         name=DISTNAME,
@@ -87,17 +139,10 @@ def read(*parts):
         install_requires=INSTALL_REQUIRES,
         setup_requires=['pytest-runner'],
         tests_require=['pytest'],
-        include_package_data=True
+        include_package_data=True,
+        package_dir={'': 'src'},
+        cmdclass={
+            'build_py': BuildWithSharedData,
+            'sdist': SDistWithSharedData
+        }
     )
-    if os.environ.get('RUNNING_ON_PI'):
-        # This only applies to software updates: when `pip install` is invoked
-        # on a running robot - not when `pip install` is invoked in the
-        # Dockerfile and not when the server starts up on a robot.
-        resource_dir = os.path.join(HERE, 'opentrons', 'resources')
-        provision = os.path.join(resource_dir, 'provision.py')
-        # We use a subprocess that invokes another python here to avoid
-        # importing the opentrons module that we’re about to install, since this
-        # is side-effect-heavy.
-        import sys
-        import subprocess
-        subprocess.check_call([sys.executable, provision], stdout=sys.stdout)

api/opentrons/config/__init__.py → api/src/opentrons/config/__init__.py

@@ -75,6 +75,7 @@ def get_config_index() -> dict:
     """
     rewrite_needed = False
     base_path = settings_dir()
+    print("Using settings dir {}".format(base_path))
     file_path = os.path.join(base_path, index_filename)
     with open(file_path) as base_config_file:
         res = json.load(base_config_file)

api/opentrons/legacy_api/instruments/pipette_config.py → api/src/opentrons/legacy_api/instruments/pipette_config.py

@@ -6,7 +6,8 @@
 
 
 root_dir = os.path.abspath(os.path.dirname(root_file))
-config_file = os.path.join(root_dir, 'config', 'pipette-config.json')
+config_file = os.path.join(root_dir,
+                           'shared_data', 'robot-data', 'pipette-config.json')
 log = logging.getLogger(__name__)
 
 pipette_config = namedtuple(