Make TensorFlow dependencies optional (#275)

Author: codesue

.github/workflows/ci.yml

@@ -28,6 +28,16 @@ jobs:
   ci:
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    strategy:
+      matrix:
+        name: [base, all]
+        include:
+          - name: base
+            dependencies: test
+            tests: model_card_toolkit --ignore-requires-optional-deps
+          - name: all
+            dependencies: all
+            tests: model_card_toolkit --fail-if-skipped
 
     steps:
     - uses: actions/checkout@v2
@@ -41,12 +51,12 @@ jobs:
         # Cache pip
         path: ~/.cache/pip
         # Check the cache for a given setup.py + MCT version
-        key: ${{ runner.os }}-pip-${{ hashFiles('model_card_toolkit/version.py') }}
+        key: ${{ runner.os }}-pip-${{ matrix.name }}-${{ hashFiles('model_card_toolkit/version.py') }}
         restore-keys: |
           ${{ runner.os }}-pip
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip wheel
-        python -m pip install -e ".[test]"
+        python -m pip install -e ".[${{ matrix.dependencies }}]"
     - name: Run tests
-      run: pytest model_card_toolkit
+      run: pytest ${{ matrix.tests }}

CODEOWNERS

@@ -8,8 +8,10 @@
 
 # PyPI distribution files (Core members)
 /model_card_toolkit/__init__.py @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
+/model_card_toolkit/dependencies.py @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
 /model_card_toolkit/version.py @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
 /package_build/ @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
+/proto_build/ @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
 /setup.py @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
 /WORKSPACE @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium
 /RELEASE.md @rcrowe-google @casassg @hanneshapke @codesue @ahmed-bd @deutranium

CONTRIBUTING.md

@@ -102,7 +102,14 @@ Follow these steps to install `model-card-toolkit`:
    pip install --upgrade pip
    ```
 
-5. Install the `model-card-toolkit` development package in editable mode:
+5. Install the `model-card-toolkit[all]` development package in editable mode:
+
+   ```sh
+   pip install -e ".[all]"
+   ```
+
+   If you only plan to make changes to parts of the codebase that don't require
+   optional dependencies, you may install `model-card-toolkit[test]` instead.
 
    ```sh
    pip install -e ".[test]"
@@ -119,7 +126,7 @@ automatically invoked once when you first install `model-card-toolkit` in
 editable mode, but further stub generation requires manual invocation.
 
 ```sh
-bazel run //model_card_toolkit:move_generated_files
+bazel run //proto_build:move_generated_files
 ```
 
 ### Licenses
@@ -142,14 +149,11 @@ with indentation width of 2 spaces.
 
 ### Linting your code
 
-Please check your code for linting errors before submitting your PR for review. Pull requests are lint checked using `pre-commit` and `pylint`.
-If you want to run the [`pre-commit`](https://pre-commit.com/) checks locally, please install
+Please check your code for linting errors before submitting your pull request for review.
+Pull requests are lint checked using [`pre-commit`](https://pre-commit.com/) and `pylint`.
+If you want to run the `pre-commit` checks locally, run following command
+from your local project directory:
 
-```sh
-pip install pre-commit
-```
-
-When you have `pre-commit` installed, you can run follow command from your local project folder to check for linting errors.
 ```sh
 pre-commit run --all-files
 ```
@@ -167,6 +171,12 @@ test coverage.
 In general, all Python files have at least one corresponding test file. For example,
 `awesome.py` should have a corresponding `awesome_test.py`.
 
+If a test suite requires an optional dependency, check [conftest.py](conftest.py)
+to verify if it can be automatically ignored with `--ignore-requires-optional-deps`
+based on its file pattern. If it can't be automatically ignored, either update
+the list of patterns to ignore in `conftest.py` or mark the tests to be skipped
+if the dependency is not installed.
+
 #### Running unit tests
 
 To run a specific test suite, e.g. `ModelCardTest`, run its test file:
@@ -175,12 +185,28 @@ To run a specific test suite, e.g. `ModelCardTest`, run its test file:
 pytest model_card_toolkit/model_card_test.py
 ```
 
-Use the following command to run all unit tests:
+To run all tests, run:
 
 ```sh
 pytest model_card_toolkit
 ```
 
+Use the following command to skip tests that require optional dependencies:
+
+```sh
+pytest model_card_toolkit --ignore-requires-optional-deps
+```
+
+Use the following command to fail skipped tests:
+
+```sh
+pytest model_card_toolkit --fail-if-skipped
+```
+
+In general, tests are marked as `skip` if they require optional dependencies
+that aren't installed. Failing skipped tests let us catch when tests are skipped
+even though all optional dependencies are installed.
+
 ## Being a Code Owner
 
 Code owners are automatically requested for review when someone opens a pull

README.md

@@ -1,6 +1,14 @@
 # Model Card Toolkit
 
-The Model Card Toolkit (MCT) streamlines and automates generation of [Model Cards](https://modelcards.withgoogle.com/about) [1], machine learning documents that provide context and transparency into a model's development and performance. Integrating the MCT into your ML pipeline enables the sharing model metadata and metrics with researchers, developers, reporters, and more.
+[![CI][ci_badge]][ci_link]
+[![PyPI][pypi_badge]][pypi_link]
+[![Documentation][docs_badge]][docs_link]
+
+The Model Card Toolkit (MCT) streamlines and automates generation of
+[Model Cards](https://modelcards.withgoogle.com/about) [1], machine learning documents
+that provide context and transparency into a model's development and performance.
+Integrating the MCT into your ML pipeline enables you to share model metadata and
+metrics with researchers, developers, reporters, and more.
 
 Some use cases of model cards include:
 
@@ -12,8 +20,27 @@ Some use cases of model cards include:
 
 ## Installation
 
-The Model Card Toolkit is hosted on [PyPI](https://pypi.org/project/model-card-toolkit/), and can be installed with `pip install model-card-toolkit` (or `pip install model-card-toolkit
---use-deprecated=legacy-resolver` for versions of pip starting with 20.3). See [the installation guide](model_card_toolkit/documentation/guide/install.md) for more details.
+The Model Card Toolkit is hosted on [PyPI](https://pypi.org/project/model-card-toolkit/),
+and requires Python 3.7 or later.
+
+Installing the basic, framework agnostic package:
+
+```sh
+pip install model-card-toolkit
+```
+
+If you are generating model cards for TensorFlow models, install the optional
+TensorFlow dependencies to use Model Card Toolkit's TensorFlow utilities:
+
+```sh
+pip install model-card-toolkit[tensorflow]
+```
+
+You may need to append the `--use-deprecated=legacy-resolver` flag when running
+versions of pip starting with 20.3.
+
+See [the installation guide](model_card_toolkit/documentation/guide/install.md)
+for more installation options.
 
 ## Getting Started
 
@@ -39,22 +66,34 @@ If you are using [TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx), yo
 incorporate model card generation into your TFX pipeline via the `ModelCardGenerator`
 component.
 
-The `ModelCardGenerator` component is moving to the
-[TFX Addons](https://github.com/tensorflow/tfx-addons) library and will no longer
-be packaged in Model Card Toolkit from version 2.0.0. Before you can use the
+The `ModelCardGenerator` component has moved to the
+[TFX Addons](https://github.com/tensorflow/tfx-addons) library and is no longer
+packaged in Model Card Toolkit from version 2.0.0. Before you can use the
 component, you will need to install the `tfx-addons` package:
 
 ```sh
 pip install tfx-addons[model_card_generator]
 ```
 
-This page will be updated to include the new links for the Model Cards in TFX
-guide and the end-to-end demo when the migration is completed.
+See the [ModelCardGenerator guide](https://github.com/tensorflow/tfx-addons/blob/main/tfx_addons/model_card_generator/README.md)
+and run the [case study notebook](https://github.com/tensorflow/tfx-addons/blob/main/examples/model_card_generator/MLMD_Model_Card_Toolkit_Demo.ipynb)
+to learn more about the component.
 
 ## Schema
 
-Model cards are stored in proto as an intermediate format. You can see the model card JSON schema in the `schema` directory.
+Model cards are stored in proto as an intermediate format. You can see the model
+card JSON schema in the `schema` directory.
 
 ## References
 
 [1] https://arxiv.org/abs/1810.03993
+
+
+[ci_badge]: https://github.com/tensorflow/model-card-toolkit/actions/workflows/ci.yml/badge.svg
+[ci_link]: https://github.com/tensorflow/model-card-toolkit/actions/workflows/ci.yml
+
+[pypi_badge]: https://badge.fury.io/py/model-card-toolkit.svg
+[pypi_link]: https://badge.fury.io/py/model-card-toolkit
+
+[docs_badge]: https://img.shields.io/badge/TensorFow-page-orange
+[docs_link]: https://www.tensorflow.org/responsible_ai/model_card_toolkit/guide

conftest.py

@@ -0,0 +1,63 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Pytest configuration."""
+
+import fnmatch
+import pathlib
+
+import pytest
+
+_REQUIRES_OPTIONAL_DEPS = ['**/tf_*_test.py']
+
+
+# Adapted from pytest-error-for-skips
+# https://github.com/jankatins/pytest-error-for-skips/blob/2.0.0/pytest_error_for_skips.py
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]):
+  """Hook to treat skipped tests as failures."""
+  outcome = yield
+  rep = outcome.get_result()
+
+  if item.config.getoption('--fail-if-skipped'):
+    if rep.skipped and call.excinfo.errisinstance(pytest.skip.Exception):
+      rep.outcome = 'failed'
+      r = call.excinfo._getreprcrash()
+      rep.longrepr = f'Fail skipped tests - {r.message}'
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_ignore_collect(
+    collection_path: pathlib.Path, config: pytest.Config
+):
+  """Hook to ignore tests that require optional dependencies."""
+  outcome = yield
+  if config.getoption('--ignore-requires-optional-deps'):
+    if any(
+        fnmatch.fnmatch(str(collection_path), pattern)
+        for pattern in _REQUIRES_OPTIONAL_DEPS
+    ):
+      outcome.force_result(True)
+
+
+def pytest_addoption(parser: pytest.Parser):
+  parser.addoption(
+      '--fail-if-skipped', action='store_true', default=False,
+      help='Treat skipped tests as failures.'
+  )
+
+  parser.addoption(
+      '--ignore-requires-optional-deps', action='store_true', default=False,
+      help='Ignore tests that require optional dependencies.'
+  )

model_card_toolkit/__init__.py

@@ -13,7 +13,39 @@
 # limitations under the License.
 """A module to streamline and automate generation of Model Cards."""
 from model_card_toolkit.core import ModelCardToolkit
-from model_card_toolkit.model_card import *
+from model_card_toolkit.model_card import (
+    Citation, ConfidenceInterval, Considerations, Dataset, Graphic,
+    GraphicsCollection, KeyVal, License, Limitation, ModelCard, ModelDetails,
+    ModelParameters, Owner, PerformanceMetric, QuantitativeAnalysis, Reference,
+    Risk, SensitiveData, Tradeoff, UseCase, User, Version
+)
 from model_card_toolkit.tfx.component import ModelCardGenerator
-from model_card_toolkit.utils.source import *
 from model_card_toolkit.version import __version__
+
+__all__ = [
+    '__version__',
+    'Citation',
+    'ConfidenceInterval',
+    'Considerations',
+    'Dataset',
+    'Graphic',
+    'GraphicsCollection',
+    'KeyVal',
+    'License',
+    'Limitation',
+    'ModelCard',
+    'ModelCardGenerator',
+    'ModelCardToolkit',
+    'ModelDetails',
+    'ModelParameters',
+    'Owner',
+    'PerformanceMetric',
+    'QuantitativeAnalysis',
+    'Reference',
+    'Risk',
+    'SensitiveData',
+    'Tradeoff',
+    'UseCase',
+    'User',
+    'Version',
+]

model_card_toolkit/base_model_card_field.py

@@ -24,7 +24,7 @@
 
 from google.protobuf import descriptor, message
 
-from model_card_toolkit.utils import validation
+from model_card_toolkit.utils import json_utils
 
 
 class BaseModelCardField(abc.ABC):
@@ -102,8 +102,9 @@ def _from_proto(self, proto: message.Message) -> "BaseModelCardField":
           setattr(self, field_name, [])
           for p in getattr(proto, field_name):
             # To get the type hint of a list is not easy.
-            field = \
-              self.__annotations__[field_name].__args__[0]()  # pytype: disable=attribute-error
+            field = (
+                self.__annotations__[field_name].__args__[0]()  # pytype: disable=attribute-error
+            )
             field._from_proto(p)  # pylint: disable=protected-access
             getattr(self, field_name).append(field)
 
@@ -136,7 +137,7 @@ def _from_json(
   ) -> "BaseModelCardField":
     """Parses a JSON dictionary into the current object."""
     for subfield_key, subfield_json_value in json_dict.items():
-      if subfield_key.startswith(validation.SCHEMA_VERSION_STRING):
+      if subfield_key.startswith(json_utils.SCHEMA_VERSION_STRING):
         continue
       elif not hasattr(field, subfield_key):
         raise ValueError(
@@ -151,8 +152,9 @@ def _from_json(
         subfield_value = []
         for item in subfield_json_value:
           if isinstance(item, dict):
-            new_object = \
-              field.__annotations__[subfield_key].__args__[0]()  # pytype: disable=attribute-error
+            new_object = (
+                field.__annotations__[subfield_key].__args__[0]()  # pytype: disable=attribute-error
+            )
             subfield_value.append(self._from_json(item, new_object))
           else:  # if primitive
             subfield_value.append(item)