Add schema validation for sample test config yaml

test/sample-test/Dockerfile

@@ -16,6 +16,7 @@ RUN pip3 install ipykernel==5.1.0
 RUN pip3 install google-api-python-client==1.7.0
 RUN pip3 install google-cloud-storage==1.17.0
 RUN pip3 install fire==0.2.1
+RUN pip3 install yamale==2.0
 
 # Install python client, including DSL compiler.
 COPY ./sdk/python /sdk/python

test/sample-test/configs/default.config.yaml

@@ -12,6 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-test_name: !!str default_sample_test # required
-# arguments: !!map optional
-test_timeout: !!int 1200 # optional
+test_name: default_sample_test
+test_timeout: 1200

test/sample-test/configs/schema.config.yaml

@@ -0,0 +1,18 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the schema for sample test config yaml file.
+test_name: str()
+arguments: map(required=False)
+test_timeout: int(min=0, required=False)

test/sample-test/configs/tfx_cab_classification.config.yaml

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-test_name: !!str tfx_cab_classification
-arguments: !!map
+test_name: tfx_cab_classification
+arguments:
   output:
   project: ml-pipeline-test
   column-names: gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/column-names.json

test/sample-test/configs/xgboost_training_cm.config.yaml

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-test_name: !!str xgboost_training_cm
-arguments: !!map
+test_name: xgboost_training_cm
+arguments:
   output:
   project: ml-pipeline-test
   train-data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv
   eval-data: gs://ml-pipeline-dataset/sample-test/sfpd/eval_5.csv
   schema: gs://ml-pipeline-dataset/sample-test/sfpd/schema.json
   rounds: 5
   workers: 2
-test_timeout: !!int 1800 # xgboost needs extra time.
+test_timeout: 1800 # xgboost needs extra time.

test/sample-test/constants.py

@@ -23,6 +23,7 @@
 TEST_DIR = os.path.join(BASE_DIR, 'test/sample-test')
 CONFIG_DIR = os.path.join(TEST_DIR, 'configs')
 DEFAULT_CONFIG = os.path.join(CONFIG_DIR, 'default.config.yaml')
+SCHEMA_CONFIG = os.path.join(CONFIG_DIR, 'schema.config.yaml')
 
 # Common test params
 RUN_LIST_PAGE_SIZE = 1000

test/sample-test/run_sample_test.py

@@ -15,10 +15,11 @@
 import os
 import tarfile
 import utils
+import yamale
 import yaml
 from datetime import datetime
 from kfp import Client
-from constants import CONFIG_DIR, DEFAULT_CONFIG
+from constants import CONFIG_DIR, DEFAULT_CONFIG, SCHEMA_CONFIG
 
 
 class PySampleChecker(object):
@@ -64,9 +65,12 @@ def check(self):
     job_name = self._testname + '_sample'
     ###### Figure out arguments from associated config files. #######
     test_args = {}
+    config_schema = yamale.make_schema(SCHEMA_CONFIG)
     try:
       with open(DEFAULT_CONFIG, 'r') as f:
         raw_args = yaml.safe_load(f)
+      default_config = yamale.make_data(DEFAULT_CONFIG)
+      yamale.validate(config_schema, default_config)  # If fails, a ValueError will be raised.
     except yaml.YAMLError as yamlerr:
       raise RuntimeError('Illegal default config:{}'.format(yamlerr))
     except OSError as ose:
@@ -76,7 +80,10 @@ def check(self):
 
     try:
       with open(os.path.join(CONFIG_DIR, '%s.config.yaml' % self._testname), 'r') as f:
-          raw_args = yaml.safe_load(f)
+        raw_args = yaml.safe_load(f)
+      test_config = yamale.make_data(os.path.join(
+          CONFIG_DIR, '%s.config.yaml' % self._testname))
+      yamale.validate(config_schema, test_config)  # If fails, a ValueError will be raised.
     except yaml.YAMLError as yamlerr:
       print('No legit yaml config file found, use default args:{}'.format(yamlerr))
     except OSError as ose: