Split out requirements from manifests from module requirements

requirements_all.txt

@@ -236,6 +236,9 @@ blockchain==1.4.4
 # bme680==1.0.5
 
 # homeassistant.components.amazon_polly
+# homeassistant.components.aws_lambda
+# homeassistant.components.aws_sns
+# homeassistant.components.aws_sqs
 # homeassistant.components.route53
 boto3==1.9.16
 

script/gen_requirements_all.py

@@ -2,12 +2,13 @@
 """Generate an updated requirements_all.txt."""
 import fnmatch
 import importlib
-import json
 import os
 import pkgutil
 import re
 import sys
 
+from script.manifest.requirements import gather_requirements_from_manifests
+
 COMMENT_REQUIREMENTS = (
     'Adafruit-DHT',
     'Adafruit_BBIO',
@@ -214,8 +215,25 @@ def gather_modules():
 
     errors = []
 
+    gather_requirements_from_manifests(process_requirements, errors, reqs)
+    gather_requirements_from_modules(errors, reqs)
+
+    for key in reqs:
+        reqs[key] = sorted(reqs[key],
+                           key=lambda name: (len(name.split('.')), name))
+
+    if errors:
+        print("******* ERROR")
+        print("Errors while importing: ", ', '.join(errors))
+        print("Make sure you import 3rd party libraries inside methods.")
+        return None
+
+    return reqs
+
+
+def gather_requirements_from_modules(errors, reqs):
+    """Collect the requirements from the modules directly."""
     for package in sorted(
-            explore_module('homeassistant.components', False) +
             explore_module('homeassistant.scripts', True) +
             explore_module('homeassistant.auth', True)):
         try:
@@ -229,51 +247,16 @@ def gather_modules():
                 errors.append(package)
             continue
 
-        manifest_path = os.path.join(
-            os.path.dirname(module.__file__),
-            'manifest.json'
-        )
-
-        manifest_file_exists = os.path.isfile(manifest_path)
-        if (not manifest_file_exists
-                and not getattr(module, 'REQUIREMENTS', None)):
-            continue
-
-        if manifest_file_exists:
-            with open(manifest_path, 'r') as manifest_file:
-                manifest = json.loads(manifest_file.read())
-
-                if not manifest.get('requirements'):
-                    continue
-                process_requirements(
-                    errors,
-                    manifest['requirements'],
-                    package,
-                    reqs
-                )
-
-        elif getattr(module, 'REQUIREMENTS', None):
+        if getattr(module, 'REQUIREMENTS', None):
             process_requirements(errors, module.REQUIREMENTS, package, reqs)
 
-    for key in reqs:
-        reqs[key] = sorted(reqs[key],
-                           key=lambda name: (len(name.split('.')), name))
-
-    if errors:
-        print("******* ERROR")
-        print("Errors while importing: ", ', '.join(errors))
-        print("Make sure you import 3rd party libraries inside methods.")
-        return None
-
-    return reqs
-
 
 def process_requirements(errors, module_requirements, package, reqs):
     """Process all of the requirements."""
     for req in module_requirements:
         if req in IGNORE_REQ:
             continue
-        if '://' in req and 'pyharmony' not in req:
+        if '://' in req:
             errors.append(
                 "{}[Only pypi dependencies are allowed: {}]".format(
                     package, req))

script/manifest/requirements.py

@@ -0,0 +1,27 @@
+"""Helpers to gather requirements from manifests."""
+from .manifest_helper import iter_manifests
+
+
+def gather_requirements_from_manifests(process_requirements, errors, reqs):
+    """Gather all of the requirements from manifests"""
+    for manifest in iter_manifests():
+        if manifest.get('domain') is None:
+            errors.append(manifest)
+            errors.append(
+                'An invalid manifest exists. Please run script/validate.py'
+            )
+            continue
+
+        if manifest.get('requirements') is None:
+            errors.append(
+                'The manifest for component {} is invalid. Please run'
+                'script/validate.py'.format(manifest['domain'])
+            )
+            continue
+
+        process_requirements(
+            errors,
+            manifest['requirements'],
+            'homeassistant.components.{}'.format(manifest['domain']),
+            reqs
+        )