Merge pull request #41080 from dotlambda/home-assistant-recursive-reqs

home-assistant: include requirements of dependencies

pkgs/servers/home-assistant/parse-requirements.py

@@ -1,28 +1,30 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ ])"
+#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests pyyaml pytz pip jinja2 voluptuous typing aiohttp async-timeout astral certifi attrs ])"
 #
-# This script downloads https://github.com/home-assistant/home-assistant/blob/master/requirements_all.txt.
-# This file contains lines of the form
+# This script downloads Home Assistant's source tarball.
+# Inside the homeassistant/components directory, each component has an associated .py file,
+# specifying required packages and other components it depends on:
 #
-#     # homeassistant.components.foo
-#     # homeassistant.components.bar
-#     foobar==1.2.3
+# REQUIREMENTS = [ 'package==1.2.3' ]
+# DEPENDENCIES = [ 'component' ]
 #
-# i.e. it lists dependencies and the components that require them.
-# By parsing the file, a dictionary mapping component to dependencies is created.
-# For all of these dependencies, Nixpkgs' python3Packages are searched for appropriate names.
+# By parsing the files, a dictionary mapping component to requirements and dependencies is created.
+# For all of these requirements and the dependencies' requirements,
+# Nixpkgs' python3Packages are searched for appropriate names.
 # Then, a Nix attribute set mapping component name to dependencies is created.
 
 from urllib.request import urlopen
-from collections import OrderedDict
+import tempfile
+from io import BytesIO
+import tarfile
+import importlib
 import subprocess
 import os
 import sys
 import json
 import re
 
-GENERAL_PREFIX = '# homeassistant.'
-COMPONENT_PREFIX = GENERAL_PREFIX + 'components.'
+COMPONENT_PREFIX = 'homeassistant.components'
 PKG_SET = 'python3Packages'
 
 # If some requirements are matched by multiple python packages,
@@ -37,28 +39,32 @@ def get_version():
         m = re.search('hassVersion = "([\\d\\.]+)";', f.read())
         return m.group(1)
 
-def fetch_reqs(version='master'):
-    requirements = {}
-    with urlopen('https://github.com/home-assistant/home-assistant/raw/{}/requirements_all.txt'.format(version)) as response:
-        components = []
-        for line in response.read().decode().splitlines():
-            if line == '':
-                components = []
-            elif line[:len(COMPONENT_PREFIX)] == COMPONENT_PREFIX:
-                component = line[len(COMPONENT_PREFIX):]
-                components.append(component)
-                if component not in requirements:
-                    requirements[component] = []
-            elif line[:len(GENERAL_PREFIX)] != GENERAL_PREFIX: # skip lines like "# homeassistant.scripts.xyz"
-                # Some dependencies are commented out because they don't build on all platforms
-                # Since they are still required for running the component, don't skip them
-                if line[:2] == '# ':
-                    line = line[2:]
-                # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
-                # Therefore, if there's a "#" in the line, only take the part after it
-                line = line[line.find('#') + 1:]
-                for component in components:
-                    requirements[component].append(line)
+def parse_components(version='master'):
+    components = {}
+    with tempfile.TemporaryDirectory() as tmp:
+        with urlopen('https://github.com/home-assistant/home-assistant/archive/{}.tar.gz'.format(version)) as response:
+            tarfile.open(fileobj=BytesIO(response.read())).extractall(tmp)
+        # Use part of a script from the Home Assistant codebase
+        sys.path.append(tmp + '/home-assistant-{}'.format(version))
+        from script.gen_requirements_all import explore_module
+        for package in explore_module(COMPONENT_PREFIX, True):
+            # Remove 'homeassistant.components.' prefix
+            component = package[len(COMPONENT_PREFIX + '.'):]
+            try:
+                module = importlib.import_module(package)
+                components[component] = {}
+                components[component]['requirements'] = getattr(module, 'REQUIREMENTS', [])
+                components[component]['dependencies'] = getattr(module, 'DEPENDENCIES', [])
+            # If there is an ImportError, the imported file is not the main file of the component
+            except ImportError:
+                continue
+    return components
+
+# Recursively get the requirements of a component and its dependencies
+def get_reqs(components, component):
+    requirements = set(components[component]['requirements'])
+    for dependency in components[component]['dependencies']:
+        requirements.update(get_reqs(components, dependency))
     return requirements
 
 # Store a JSON dump of Nixpkgs' python3Packages
@@ -95,11 +101,14 @@ def name_to_attr_path(req):
 
 version = get_version()
 print('Generating component-packages.nix for version {}'.format(version))
-requirements = fetch_reqs(version=version)
+components = parse_components(version=version)
 build_inputs = {}
-for component, reqs in OrderedDict(sorted(requirements.items())).items():
+for component in sorted(components.keys()):
     attr_paths = []
-    for req in reqs:
+    for req in sorted(get_reqs(components, component)):
+        # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
+        # Therefore, if there's a "#" in the line, only take the part after it
+        req = req[req.find('#') + 1:]
         name = req.split('==')[0]
         attr_path = name_to_attr_path(name)
         if attr_path is not None:
@@ -108,11 +117,8 @@ for component, reqs in OrderedDict(sorted(requirements.items())).items():
     else:
         build_inputs[component] = attr_paths
 
-# Only select components which have any dependency
-#build_inputs = {k: v for k, v in build_inputs.items() if len(v) > 0}
-
 with open(os.path.dirname(sys.argv[0]) + '/component-packages.nix', 'w') as f:
-    f.write('# Generated from parse-requirements.py\n')
+    f.write('# Generated by parse-requirements.py\n')
     f.write('# Do not edit!\n\n')
     f.write('{\n')
     f.write('  version = "{}";\n'.format(version))