Consolidate dependencies (quantumlib#4131)

Moves all dependency logic to requirements.txt file composition rules. Setting up your local env now is just a single line: `pip install -r dev_tools/requirements/dev.env.txt` Each job in CI is referring to a single pip requirements file ,e.g. `dev_tools/requirements/mypy.env.txt` - which in turn includes -`./deps/cirq-all.txt` and `./deps/mypy.txt`. `dev_tools/requirements/deps/cirq-all.txt` is a single place to define all the dependencies for all cirq modules. This will mainly help with the issues listed in quantumlib#3996 as - it centralizes the list of cirq modules in a single place, - removes the 'cat req.txt | grep pytest | xargs pip install` type of instructions that are error-prone and slow (separate calls to pip install vs a single call for all the requirements) All in all, it will help with making our scripts more scalable as more modules are extracted.
dabacon · May 28, 2021 · 836bbda · 836bbda
1 parent b28b163
commit 836bbda
Show file tree

Hide file tree

Showing 35 changed files with 282 additions and 121 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,10 +25,8 @@ jobs:
         with:
           python-version: '3.7'
           architecture: 'x64'
-      - name: Install flynt
-        run: cat dev_tools/conf/pip-list-dev-tools.txt | grep flynt | xargs pip install
-      - name: Install black
-        run: cat dev_tools/conf/pip-list-dev-tools.txt | grep black | xargs pip install
+      - name: Install dependencies
+        run: pip install -r dev_tools/requirements/deps/format.txt
       - name: Format
         run: check/format-incremental
   mypy:
@@ -41,9 +39,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install mypy
-        run: cat dev_tools/conf/pip-list-dev-tools.txt | grep mypy | xargs pip install
-      - name: Install dependencies
-        run: pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
+        run: pip install -r dev_tools/requirements/mypy.env.txt
       - name: Type check
         run: check/mypy
   mypy-next:
@@ -56,9 +52,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install mypy
-        run: cat dev_tools/conf/pip-list-dev-tools.txt | grep mypy | xargs pip install
-      - name: Install dependencies
-        run: pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
+        run: pip install -r dev_tools/requirements/mypy.env.txt
       - name: Type check (for logging only)
         run: check/mypy --next || true
   changed_files:
@@ -86,7 +80,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install pylint
-        run: cat dev_tools/conf/pip-list-dev-tools.txt | grep "pylint" | grep -v "#" | xargs pip install
+        run: pip install -r dev_tools/requirements/deps/pylint.txt
       - name: Display version
         run: check/pylint --version
       - name: Lint
@@ -101,7 +95,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install requirements
-        run: pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt -r cirq-core/cirq/contrib/contrib-requirements.txt -r dev_tools/conf/pip-list-dev-tools.txt
+        run: pip install -r dev_tools/requirements/dev.env.txt
       - name: RST check
         run: find . -type f -name "*.rst" | xargs rstcheck --report warning --ignore-directives autoclass,automodule
       - name: Doc check
@@ -127,11 +121,7 @@ jobs:
           python-version: '3.8'
           architecture: 'x64'
       - name: Install requirements
-        run: |
-          pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-          pip install -r dev_tools/conf/pip-list-dev-tools.txt
-          # TODO: move to requirements.txt after #3704
-          pip install codeowners==0.1.2 # linux only package
+        run: pip install -r dev_tools/requirements/cirq-only.env.txt
       - name: Pytest check
         run: check/pytest --cirq-only --ignore=cirq-core/cirq/contrib --actually-quiet
   pytest:
@@ -152,10 +142,7 @@ jobs:
         run: pip install pip==20.2
       - name: Install requirements
         run: |
-          pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-          pip install -r dev_tools/conf/pip-list-dev-tools.txt
-          # TODO: move to requirements.txt after #3704
-          pip install codeowners==0.1.2 # linux only package
+          pip install -r dev_tools/requirements/dev.env.txt
       - name: Pytest check
         run: check/pytest --ignore=cirq-core/cirq/contrib --actually-quiet
   build_docs:
@@ -185,7 +172,7 @@ jobs:
           architecture: 'x64'
       - name: Install requirements
         run: |
-          python -m pip install -r dev_tools/conf/pip-list-dev-tools.txt
+          pip install -r dev_tools/requirements/deps/protos.txt
           wget https://github.com/bazelbuild/bazel/releases/download/0.26.0/bazel_0.26.0-linux-x86_64.deb
           sudo dpkg -i bazel_0.26.0-linux-x86_64.deb
       - name: Build protos
@@ -202,10 +189,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install requirements
-        run: |
-          pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-          pip install -r cirq-core/cirq/contrib/contrib-requirements.txt
-          pip install -r dev_tools/conf/pip-list-dev-tools.txt
+        run: pip install -r dev_tools/requirements/dev.env.txt
       - name: Coverage check
         run: check/pytest-and-incremental-coverage --actually-quiet
   windows:
@@ -221,9 +205,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: 'x64'
       - name: Install requirements
-        run: |
-          pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-          pip install -r dev_tools/conf/pip-list-dev-tools.txt
+        run: pip install -r dev_tools/requirements/no-contrib.env.txt
       - name: Pytest Windows
         run: check/pytest --ignore=cirq-core/cirq/contrib --actually-quiet
         shell: bash
@@ -240,9 +222,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: 'x64'
       - name: Install requirements
-        run: |
-          pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-          pip install -r dev_tools/conf/pip-list-dev-tools.txt
+        run: pip install -r dev_tools/requirements/no-contrib.env.txt
       - name: Pytest check
         run: check/pytest --ignore=cirq-core/cirq/contrib
   notebooks-stable:
@@ -257,7 +237,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install requirements
-        run: pip install -r dev_tools/notebooks/requirements-isolated-notebook-tests.txt
+        run: pip install -r dev_tools/requirements/isolated-notebooks.env.txt
       - name: Notebook tests
         run: check/pytest -n auto -m slow dev_tools/notebooks/isolated_notebook_test.py
       - uses: actions/upload-artifact@v2
@@ -275,7 +255,7 @@ jobs:
           python-version: '3.7'
           architecture: 'x64'
       - name: Install requirements
-        run: pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt -r cirq-core/cirq/contrib/contrib-requirements.txt -r dev_tools/notebooks/requirements-notebook-tests.txt
+        run: pip install -r dev_tools/requirements/notebooks.env.txt
       - name: Notebook tests
         run: check/pytest -n auto -m slow dev_tools/notebooks/notebook_test.py
       - uses: actions/upload-artifact@v2

diff --git a/check/nbformat b/check/nbformat
@@ -28,14 +28,6 @@ done
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$(git rev-parse --show-toplevel)"
 
-# Install a pinned version of tf-docs
-TF_DOCS_COMMIT=b3dc8a922d8bdc6e998a8ad4f4953359dc6e576a
-TF_DOCS_INSTALLED=$(pip list | grep  -c $TF_DOCS_COMMIT)
-
-if [ "$TF_DOCS_INSTALLED" != "1" ]; then
-  pip install -U git+https://github.com/tensorflow/docs@$TF_DOCS_COMMIT
-fi
-
 FORMAT_CMD="python3 -m tensorflow_docs.tools.nbfmt --indent=1"
 
 # Test the notebooks

diff --git a/...ore/cirq/contrib/contrib-requirements.txt → cirq-core/cirq/contrib/requirements.txt b/...ore/cirq/contrib/contrib-requirements.txt → cirq-core/cirq/contrib/requirements.txt
diff --git a/dev_tools/conf/pip-install-minimal-for-pytest-changed-files.sh b/dev_tools/conf/pip-install-minimal-for-pytest-changed-files.sh
@@ -20,19 +20,10 @@ set -e
 cd "$( dirname "${BASH_SOURCE[0]}" )"
 cd "$(git rev-parse --show-toplevel)"
 
-# Install usual requirements.
-pip install -r cirq-core/requirements.txt -r cirq-google/requirements.txt
-
-# Install pytest related dev requirements.
-cat dev_tools/conf/pip-list-dev-tools.txt | grep pytest | xargs pip install
-cat dev_tools/conf/pip-list-dev-tools.txt | grep filelock | xargs pip install
-cat dev_tools/conf/pip-list-dev-tools.txt | grep freezegun | xargs pip install
-cat dev_tools/conf/pip-list-dev-tools.txt | grep flynt | xargs pip install
-cat dev_tools/conf/pip-list-dev-tools.txt | grep importlib-metadata | xargs -0 pip install
-
-# TODO: move to requirements.txt after #3704
-pip install codeowners==0.1.2 # linux only package
+REQS="-r dev_tools/requirements/pytest-minimal.env.txt"
 
 # Install contrib requirements only if needed.
 changed=$(git diff --name-only origin/master | grep "cirq/contrib" || true)
-[ "${changed}" = "" ] || pip install -r cirq-core/cirq/contrib/contrib-requirements.txt
+[ "${changed}" = "" ] || REQS="$REQS -r cirq-core/cirq/contrib/requirements.txt"
+
+pip install $REQS
diff --git a/dev_tools/conf/pip-list-dev-tools.txt b/dev_tools/conf/pip-list-dev-tools.txt
diff --git a/dev_tools/env_tools.py b/dev_tools/env_tools.py
@@ -110,8 +110,8 @@ def prepare_temporary_test_environment(
     base_path = cast(str, env.destination_directory)
     env_path = os.path.join(base_path, env_name)
     req_path = os.path.join(base_path, 'requirements.txt')
-    dev_req_path = os.path.join(base_path, 'dev_tools', 'conf', 'pip-list-dev-tools.txt')
-    contrib_req_path = os.path.join(base_path, 'cirq', 'contrib', 'contrib-requirements.txt')
+    dev_req_path = os.path.join(base_path, 'dev_tools', 'requirements', 'deps', 'dev-tools.txt')
+    contrib_req_path = os.path.join(base_path, 'cirq', 'contrib', 'requirements.txt')
     rev_paths = [req_path, dev_req_path, contrib_req_path]
     create_virtual_env(
         venv_path=env_path, python_path=python_path, requirements_paths=rev_paths, verbose=verbose

diff --git a/dev_tools/notebooks/isolated_notebook_test.py b/dev_tools/notebooks/isolated_notebook_test.py
@@ -96,7 +96,8 @@ def _list_changed_notebooks() -> Set[str]:
         rev = _find_base_revision()
         output = subprocess.check_output(f'git diff --name-only {rev}'.split())
         lines = output.decode('utf-8').splitlines()
-        if any(l for l in lines if l.endswith("isolated_notebook_test.py")):
+        # run all tests if this file or any of the dependencies change
+        if any(l for l in lines if l.endswith("isolated_notebook_test.py") or l.endswith(".txt")):
             return list_all_notebooks()
         return set(l for l in lines if l.endswith(".ipynb"))
     except ValueError as e:

diff --git a/dev_tools/notebooks/requirements-isolated-notebook-tests.txt b/dev_tools/notebooks/requirements-isolated-notebook-tests.txt
diff --git a/dev_tools/packaging/verify-published-package.sh b/dev_tools/packaging/verify-published-package.sh
@@ -75,7 +75,7 @@ for PYTHON_VERSION in python3; do
     CORE_DEPS_FILE="${REPO_ROOT}/cirq-core/requirements.txt"
     GOOGLE_DEPS_FILE="${REPO_ROOT}/cirq-google/requirements.txt"
     CONTRIB_DEPS_FILE="${REPO_ROOT}/cirq-core/cirq/contrib/contrib-requirements.txt"
-    DEV_DEPS_FILE="${REPO_ROOT}/dev_tools/conf/pip-list-dev-tools.txt"
+    DEV_DEPS_FILE="${REPO_ROOT}/dev_tools/requirements/deps/dev-tools.txt"
 
     echo -e "\n\033[32m${PYTHON_VERSION}\033[0m"
     echo "Working in a fresh virtualenv at ${tmp_dir}/${PYTHON_VERSION}"

diff --git a/dev_tools/requirements/__init__.py b/dev_tools/requirements/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dev_tools.requirements.reqs import explode
diff --git a/dev_tools/requirements/cirq-only.env.txt b/dev_tools/requirements/cirq-only.env.txt
@@ -0,0 +1,2 @@
+-r ../../cirq-core/requirements.txt
+-r deps/dev-tools.txt
diff --git a/dev_tools/requirements/deps/cirq-all-no-contrib.txt b/dev_tools/requirements/deps/cirq-all-no-contrib.txt
@@ -0,0 +1,3 @@
+# captures all the modules
+-r ../../../cirq-core/requirements.txt
+-r ../../../cirq-google/requirements.txt
diff --git a/dev_tools/requirements/deps/cirq-all.txt b/dev_tools/requirements/deps/cirq-all.txt
@@ -0,0 +1,4 @@
+# captures all the modules, including contrib
+
+-r cirq-all-no-contrib.txt
+-r ../../../cirq-core/cirq/contrib/requirements.txt
diff --git a/dev_tools/requirements/deps/dev-tools.txt b/dev_tools/requirements/deps/dev-tools.txt
@@ -0,0 +1,22 @@
+-r mypy.txt
+-r pytest.txt
+-r format.txt
+-r pylint.txt
+-r protos.txt
+-r notebook.txt
+
+# For testing and analyzing code.
+asv
+virtualenv
+
+# For uploading packages to pypi.
+twine
+
+# For verifying behavior of Quil output.
+pyquil~=2.21.0
+
+# For verifying behavior of qasm output.
+qiskit-aer~=0.7.6
+
+# For verifying rst
+rstcheck~=3.3.1
diff --git a/dev_tools/requirements/deps/format.txt b/dev_tools/requirements/deps/format.txt
@@ -0,0 +1,2 @@
+flynt~=0.60
+black==20.8b1
diff --git a/dev_tools/requirements/deps/mypy.txt b/dev_tools/requirements/deps/mypy.txt
@@ -0,0 +1,2 @@
+# the mypy dependency file
+mypy~=0.782.0
diff --git a/dev_tools/requirements/deps/nbformat.txt b/dev_tools/requirements/deps/nbformat.txt
@@ -0,0 +1,2 @@
+# TODO: add latest commit retrieval to compile.py - pip-compile can't upgrade automatically
+git+https://github.com/tensorflow/docs@b3dc8a922d8bdc6e998a8ad4f4953359dc6e576a
diff --git a/...notebooks/requirements-notebook-tests.txt → dev_tools/requirements/deps/notebook.txt b/...notebooks/requirements-notebook-tests.txt → dev_tools/requirements/deps/notebook.txt
@@ -1,17 +1,14 @@
-pytest>=6.0.0
-pytest-xdist~=2.2.0
-filelock~=3.0.12
-
-papermill~=2.3.2
 notebook~=6.2.0
 
 # https://github.com/nteract/papermill/issues/519
 ipykernel==5.3.4
 
 # https://github.com/ipython/ipython/issues/12941
-ipython==7.22
+ipython==7.22; python_version >= '3.7'
+ipython; python_version < '3.7'
 
-# assumed to be part of colab
-seaborn~=0.11.1
+# for executing notebooks in tests
+papermill~=2.3.2
 
-ipython==7.22
+# assumed to be part of colab
+seaborn~=0.11.1
diff --git a/dev_tools/requirements/deps/protos.txt b/dev_tools/requirements/deps/protos.txt
@@ -0,0 +1,8 @@
+# dependencies for proto generation - used by cirq-google
+
+# 1.26.0 is the min version that has python 3.8 wheels
+# however, we can't really go much higher - the protoc version needs to be <= 3.9.2
+# otherwise it will generate pb2 files that are incompatible with tensorflow quantum
+grpcio-tools~=1.26.0
+
+mypy-protobuf==1.10
diff --git a/dev_tools/requirements/deps/pylint.txt b/dev_tools/requirements/deps/pylint.txt
@@ -0,0 +1,3 @@
+# for linting
+
+pylint~=2.6.0
diff --git a/dev_tools/requirements/deps/pytest.txt b/dev_tools/requirements/deps/pytest.txt
@@ -0,0 +1,18 @@
+# dependencies to run pytest
+
+pytest
+pytest-asyncio
+pytest-cov
+
+# for parallel testing notebooks
+pytest-xdist~=2.2.0
+filelock~=3.0.12
+
+# For testing time specific logic
+freezegun~=0.3.15
+
+# for python 3.7 and below needs to be installed
+importlib-metadata; python_version < '3.8'
+
+# codeowners test dependency
+codeowners; python_version >= '3.7'
diff --git a/dev_tools/requirements/dev.env.txt b/dev_tools/requirements/dev.env.txt
@@ -0,0 +1,7 @@
+# This file describes a full blown development environment with cirq
+# It installs the dependencies of all the modules
+# compile it with pip-compile dev_tools/requirements/dev.txt
+
+-r deps/cirq-all.txt
+-r deps/dev-tools.txt
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-r ../../cirq-core/requirements.txt
		-r deps/dev-tools.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# TODO: add latest commit retrieval to compile.py - pip-compile can't upgrade automatically
		git+https://github.com/tensorflow/docs@b3dc8a922d8bdc6e998a8ad4f4953359dc6e576a