Merge pull request dbt-labs#1046 from fishtown-analytics/feature/pars…

…e-csv-seeds-only

Only parse CSV files during "dbt seed" (dbt-labs#867)

dbt/config.py

@@ -677,9 +677,10 @@ def __init__(self, project_name, version, project_root, source_paths,
                  modules_path, quoting, models, on_run_start, on_run_end,
                  archive, seeds, profile_name, target_name,
                  send_anonymous_usage_stats, use_colors, threads, credentials,
-                 packages, cli_vars):
+                 packages, args):
         # 'vars'
-        self.cli_vars = cli_vars
+        self.args = args
+        self.cli_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}'))
         # 'project'
         Project.__init__(
             self,
@@ -718,13 +719,12 @@ def __init__(self, project_name, version, project_root, source_paths,
         self.validate()
 
     @classmethod
-    def from_parts(cls, project, profile, cli_vars):
+    def from_parts(cls, project, profile, args):
         """Instantiate a RuntimeConfig from its components.
 
         :param profile Profile: A parsed dbt Profile.
         :param project Project: A parsed dbt Project.
-        :param cli_vars dict: A dict of vars, as provided from the command
-            line.
+        :param args argparse.Namespace: The parsed command-line arguments.
         :returns RuntimeConfig: The new configuration.
         """
         quoting = deepcopy(
@@ -759,7 +759,7 @@ def from_parts(cls, project, profile, cli_vars):
             use_colors=profile.use_colors,
             threads=profile.threads,
             credentials=profile.credentials,
-            cli_vars=cli_vars
+            args=args
         )
 
     def new_project(self, project_root):
@@ -780,7 +780,7 @@ def new_project(self, project_root):
         cfg = self.from_parts(
             project=project,
             profile=profile,
-            cli_vars=deepcopy(self.cli_vars)
+            args=deepcopy(self.args),
         )
         # force our quoting back onto the new project.
         cfg.quoting = deepcopy(self.quoting)
@@ -791,6 +791,8 @@ def serialize(self):
         instance that has passed validate() (which happens in __init__), it
         matches the Configuration contract.
 
+        Note that args are not serialized.
+
         :returns dict: The serialized configuration.
         """
         result = self.to_project_config(with_packages=True)
@@ -837,7 +839,7 @@ def from_args(cls, args):
         return cls.from_parts(
             project=project,
             profile=profile,
-            cli_vars=cli_vars
+            args=args
         )
 
 

dbt/parser/seeds.py

@@ -15,7 +15,7 @@
 
 class SeedParser(BaseParser):
     @classmethod
-    def parse_seed_file(cls, file_match, root_dir, package_name):
+    def parse_seed_file(cls, file_match, root_dir, package_name, should_parse):
         """Parse the given seed file, returning an UnparsedNode and the agate
         table.
         """
@@ -34,10 +34,13 @@ def parse_seed_file(cls, file_match, root_dir, package_name):
             original_file_path=os.path.join(file_match.get('searched_path'),
                                             file_match.get('relative_path')),
         )
-        try:
-            table = dbt.clients.agate_helper.from_csv(abspath)
-        except ValueError as e:
-            dbt.exceptions.raise_compiler_error(str(e), node)
+        if should_parse:
+            try:
+                table = dbt.clients.agate_helper.from_csv(abspath)
+            except ValueError as e:
+                dbt.exceptions.raise_compiler_error(str(e), node)
+        else:
+            table = dbt.clients.agate_helper.empty_table()
         table.original_abspath = abspath
         return node, table
 
@@ -56,10 +59,13 @@ def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
             relative_dirs,
             extension)
 
+        # we only want to parse seeds if we're inside 'dbt seed'
+        should_parse = root_project.args.which == 'seed'
+
         result = {}
         for file_match in file_matches:
             node, agate_table = cls.parse_seed_file(file_match, root_dir,
-                                                    package_name)
+                                                    package_name, should_parse)
             node_path = cls.get_path(NodeType.Seed, package_name, node.name)
             parsed = cls.parse_node(node, node_path, root_project,
                                     all_projects.get(package_name),

test/integration/005_simple_seed_test/data-bad/seed.csv

@@ -0,0 +1,3 @@
+a,b,c
+1,7,23,90,5
+2

test/integration/005_simple_seed_test/models-exist/model.sql

@@ -0,0 +1 @@
+select * from {{ this.schema }}.seed_expected

test/integration/005_simple_seed_test/test_simple_seed.py

@@ -1,6 +1,8 @@
 from nose.plugins.attrib import attr
 from test.integration.base import DBTIntegrationTest
 
+from dbt.exceptions import CompilationException
+
 class TestSimpleSeed(DBTIntegrationTest):
 
     def setUp(self):
@@ -129,3 +131,36 @@ def test_simple_seed_with_disabled(self):
         self.assertEqual(len(results),  1)
         self.assertTableDoesExist('seed_enabled')
         self.assertTableDoesNotExist('seed_disabled')
+
+
+class TestSeedParsing(DBTIntegrationTest):
+    def setUp(self):
+        super(TestSeedParsing, self).setUp()
+        self.run_sql_file("test/integration/005_simple_seed_test/seed.sql")
+
+    @property
+    def schema(self):
+        return "simple_seed_005"
+
+    @property
+    def models(self):
+        return "test/integration/005_simple_seed_test/models-exist"
+
+    @property
+    def project_config(self):
+        return {
+            "data-paths": ['test/integration/005_simple_seed_test/data-bad']
+        }
+
+    @attr(type='postgres')
+    def test_postgres_dbt_run_skips_seeds(self):
+        # run does not try to parse the seed files
+        self.assertEqual(len(self.run_dbt(['run'])), 1)
+
+        # make sure 'dbt seed' fails, otherwise our test is invalid!
+        with self.assertRaises(CompilationException):
+            self.run_dbt(['seed'])
+
+
+
+

test/integration/base.py

@@ -38,6 +38,7 @@ def __init__(self):
 
 class TestArgs(object):
     def __init__(self, kwargs):
+        self.which = 'run'
         self.__dict__.update(kwargs)
 
 

test/unit/utils.py

@@ -4,6 +4,10 @@
 issues.
 """
 
+class Obj(object):
+    which = 'blah'
+
+
 def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
     from dbt.config import Project, Profile, RuntimeConfig
     from dbt.utils import parse_cli_vars
@@ -16,9 +20,10 @@ def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
         profile = Profile.from_raw_profile_info(deepcopy(profile),
                                                 project.profile_name,
                                                 cli_vars)
-
+    args = Obj()
+    args.cli_vars = cli_vars
     return RuntimeConfig.from_parts(
         project=project,
         profile=profile,
-        cli_vars=cli_vars
+        args=args
     )