mirror of
https://github.com/SebastianWendel/nixpkgs.git
synced 2024-09-20 04:19:00 +02:00
Merge pull request #188306 from gbpdt/fix/airflow_2.3.4
This commit is contained in:
Martin Weinelt
GitHub
commit
a58203d752
|
|
@ -4788,6 +4788,12 @@
|
|||
githubId = 16470252;
|
||||
name = "Gemini Lasswell";
|
||||
};
|
||||
gbpdt = {
|
||||
email = "nix@pdtpartners.com";
|
||||
github = "gbpdt";
|
||||
githubId = 25106405;
|
||||
name = "Graham Bennett";
|
||||
};
|
||||
gbtb = {
|
||||
email = "goodbetterthebeast3@gmail.com";
|
||||
github = "gbtb";
|
||||
|
|
|
|||
|
|
@ -11,15 +11,19 @@
|
|||
, cattrs
|
||||
, clickclick
|
||||
, colorlog
|
||||
, connexion
|
||||
, cron-descriptor
|
||||
, croniter
|
||||
, cryptography
|
||||
, dataclasses
|
||||
, deprecated
|
||||
, dill
|
||||
, flask
|
||||
, flask_login
|
||||
, flask-wtf
|
||||
, flask-appbuilder
|
||||
, flask-caching
|
||||
, flask-session
|
||||
, flask-wtf
|
||||
, GitPython
|
||||
, graphviz
|
||||
, gunicorn
|
||||
|
|
@ -32,13 +36,16 @@
|
|||
, jinja2
|
||||
, jsonschema
|
||||
, lazy-object-proxy
|
||||
, linkify-it-py
|
||||
, lockfile
|
||||
, markdown
|
||||
, markupsafe
|
||||
, marshmallow-oneofschema
|
||||
, mdit-py-plugins
|
||||
, numpy
|
||||
, openapi-spec-validator
|
||||
, pandas
|
||||
, pathspec
|
||||
, pendulum
|
||||
, psutil
|
||||
, pygments
|
||||
|
|
@ -58,20 +65,27 @@
|
|||
, tabulate
|
||||
, tenacity
|
||||
, termcolor
|
||||
, typing-extensions
|
||||
, unicodecsv
|
||||
, werkzeug
|
||||
, pytestCheckHook
|
||||
, freezegun
|
||||
, mkYarnPackage
|
||||
, writeScript
|
||||
|
||||
# Extra airflow providers to enable
|
||||
, enabledProviders ? []
|
||||
}:
|
||||
let
|
||||
version = "2.3.3";
|
||||
version = "2.3.4";
|
||||
|
||||
airflow-src = fetchFromGitHub rec {
|
||||
owner = "apache";
|
||||
repo = "airflow";
|
||||
rev = "refs/tags/${version}";
|
||||
sha256 = "sha256-N+6ljfSo6+UvSAnvDav6G0S49JZ1VJwxmaiKPV3/DjA=";
|
||||
# Required because the GitHub archive tarballs don't appear to include tests
|
||||
leaveDotGit = true;
|
||||
sha256 = "sha256-rxvLyz/hvZ6U8QKy9MiVofU0qeeo7OHctAj2PkxLh2c=";
|
||||
};
|
||||
|
||||
# airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
|
||||
|
|
@ -87,6 +101,12 @@ let
|
|||
|
||||
distPhase = "true";
|
||||
|
||||
# The webpack license plugin tries to create /licenses when given the
|
||||
# original relative path
|
||||
postPatch = ''
|
||||
sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
|
||||
'';
|
||||
|
||||
configurePhase = ''
|
||||
cp -r $node_modules node_modules
|
||||
'';
|
||||
|
|
@ -102,6 +122,13 @@ let
|
|||
'';
|
||||
};
|
||||
|
||||
# Import generated file with metadata for provider dependencies and imports.
|
||||
# Enable additional providers using enabledProviders above.
|
||||
providers = import ./providers.nix;
|
||||
getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
|
||||
getProviderImports = provider: providers.${provider}.imports;
|
||||
providerDependencies = lib.concatMap getProviderDeps enabledProviders;
|
||||
providerImports = lib.concatMap getProviderImports enabledProviders;
|
||||
in
|
||||
buildPythonPackage rec {
|
||||
pname = "apache-airflow";
|
||||
|
|
@ -119,14 +146,18 @@ buildPythonPackage rec {
|
|||
cattrs
|
||||
clickclick
|
||||
colorlog
|
||||
connexion
|
||||
cron-descriptor
|
||||
croniter
|
||||
cryptography
|
||||
deprecated
|
||||
dill
|
||||
flask
|
||||
flask-appbuilder
|
||||
flask-caching
|
||||
flask_login
|
||||
flask-session
|
||||
flask-wtf
|
||||
flask_login
|
||||
GitPython
|
||||
graphviz
|
||||
gunicorn
|
||||
|
|
@ -138,13 +169,16 @@ buildPythonPackage rec {
|
|||
jinja2
|
||||
jsonschema
|
||||
lazy-object-proxy
|
||||
linkify-it-py
|
||||
lockfile
|
||||
markdown
|
||||
markupsafe
|
||||
marshmallow-oneofschema
|
||||
mdit-py-plugins
|
||||
numpy
|
||||
openapi-spec-validator
|
||||
pandas
|
||||
pathspec
|
||||
pendulum
|
||||
psutil
|
||||
pygments
|
||||
|
|
@ -163,13 +197,14 @@ buildPythonPackage rec {
|
|||
tabulate
|
||||
tenacity
|
||||
termcolor
|
||||
typing-extensions
|
||||
unicodecsv
|
||||
werkzeug
|
||||
] ++ lib.optionals (pythonOlder "3.7") [
|
||||
dataclasses
|
||||
] ++ lib.optionals (pythonOlder "3.9") [
|
||||
importlib-metadata
|
||||
];
|
||||
] ++ providerDependencies;
|
||||
|
||||
buildInputs = [
|
||||
airflow-frontend
|
||||
|
|
@ -180,29 +215,15 @@ buildPythonPackage rec {
|
|||
pytestCheckHook
|
||||
];
|
||||
|
||||
# By default, source code of providers is included but unusable due to missing
|
||||
# transitive dependencies. To enable a provider, add it to extraProviders
|
||||
# above
|
||||
INSTALL_PROVIDERS_FROM_SOURCES = "true";
|
||||
|
||||
postPatch = ''
|
||||
substituteInPlace setup.cfg \
|
||||
--replace "attrs>=20.0, <21.0" "attrs" \
|
||||
--replace "cattrs~=1.1, <1.7.0" "cattrs" \
|
||||
--replace "colorlog>=4.0.2, <6.0" "colorlog" \
|
||||
--replace "croniter>=0.3.17, <1.1" "croniter" \
|
||||
--replace "docutils<0.17" "docutils" \
|
||||
--replace "flask-login>=0.3, <0.5" "flask-login" \
|
||||
--replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
|
||||
--replace "flask>=1.1.0, <2.0" "flask" \
|
||||
--replace "importlib_resources~=1.4" "importlib_resources" \
|
||||
--replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \
|
||||
--replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
|
||||
--replace "pyjwt<2" "pyjwt" \
|
||||
--replace "python-slugify>=3.0.0,<5.0" "python-slugify" \
|
||||
--replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
|
||||
--replace "tenacity~=6.2.0" "tenacity" \
|
||||
--replace "werkzeug~=1.0, >=1.0.1" "werkzeug"
|
||||
|
||||
substituteInPlace tests/core/test_core.py \
|
||||
--replace "/bin/bash" "${stdenv.shell}"
|
||||
--replace "colorlog>=4.0.2, <5.0" "colorlog" \
|
||||
--replace "flask-login>=0.6.2" "flask-login"
|
||||
'' + lib.optionalString stdenv.isDarwin ''
|
||||
# Fix failing test on Hydra
|
||||
substituteInPlace airflow/utils/db.py \
|
||||
|
|
@ -214,7 +235,11 @@ buildPythonPackage rec {
|
|||
"--prefix PYTHONPATH : $PYTHONPATH"
|
||||
];
|
||||
|
||||
preCheck = ''
|
||||
pythonImportsCheck = [
|
||||
"airflow"
|
||||
] ++ providerImports;
|
||||
|
||||
checkPhase = ''
|
||||
export HOME=$(mktemp -d)
|
||||
export AIRFLOW_HOME=$HOME
|
||||
export AIRFLOW__CORE__UNIT_TEST_MODE=True
|
||||
|
|
@ -238,12 +263,37 @@ buildPythonPackage rec {
|
|||
cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
|
||||
'';
|
||||
|
||||
# Updates yarn.lock and package.json
|
||||
passthru.updateScript = writeScript "update.sh" ''
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Get new version
|
||||
new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
|
||||
pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
|
||||
update-source-version ${pname} "$new_version"
|
||||
|
||||
# Update frontend
|
||||
cd ./pkgs/development/python-modules/apache-airflow
|
||||
curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
|
||||
curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
|
||||
# Note: for 2.3.4 a manual change was needed to get a fully resolved URL for
|
||||
# caniuse-lite@1.0.30001312 (with the sha after the #). The error from yarn
|
||||
# was 'Can't make a request in offline mode' from yarn. Corrected install by
|
||||
# manually running yarn add caniuse-lite@1.0.30001312 and copying the
|
||||
# requisite section from the generated yarn.lock.
|
||||
yarn2nix > yarn.nix
|
||||
|
||||
# update provider dependencies
|
||||
./update-providers.py
|
||||
'';
|
||||
|
||||
meta = with lib; {
|
||||
description = "Programmatically author, schedule and monitor data pipelines";
|
||||
homepage = "https://airflow.apache.org/";
|
||||
license = licenses.asl20;
|
||||
maintainers = with maintainers; [ bhipple costrouc ingenieroariel ];
|
||||
# requires extremely outdated versions of multiple dependencies
|
||||
broken = true;
|
||||
maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
{
|
||||
"name": "airflow-frontend",
|
||||
"version": "2.1.1rc1",
|
||||
"name": "airflow-www",
|
||||
"version": "1.0.0",
|
||||
"description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development",
|
||||
"prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress",
|
||||
"build": "NODE_ENV=production webpack --colors --progress",
|
||||
"lint": "eslint --ignore-path=.eslintignore --ext .js,.html .",
|
||||
"lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ."
|
||||
"test": "jest",
|
||||
"dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development",
|
||||
"prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress",
|
||||
"build": "NODE_ENV=production webpack --progress --mode production",
|
||||
"lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit",
|
||||
"lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit"
|
||||
},
|
||||
"author": "Apache",
|
||||
"license": "Apache-2.0",
|
||||
|
|
@ -28,53 +28,82 @@
|
|||
"flask"
|
||||
],
|
||||
"devDependencies": {
|
||||
"babel": "^6.23.0",
|
||||
"babel-core": "^6.26.3",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"@babel/core": "^7.18.5",
|
||||
"@babel/eslint-parser": "^7.18.2",
|
||||
"@babel/plugin-transform-runtime": "^7.16.0",
|
||||
"@babel/preset-env": "^7.16.0",
|
||||
"@babel/preset-react": "^7.16.0",
|
||||
"@babel/preset-typescript": "^7.17.12",
|
||||
"@testing-library/jest-dom": "^5.16.0",
|
||||
"@testing-library/react": "^13.0.0",
|
||||
"@types/react": "^18.0.12",
|
||||
"@types/react-dom": "^18.0.5",
|
||||
"@typescript-eslint/eslint-plugin": "^5.13.0",
|
||||
"@typescript-eslint/parser": "^5.0.0",
|
||||
"babel-jest": "^27.3.1",
|
||||
"babel-loader": "^8.1.0",
|
||||
"babel-plugin-css-modules-transform": "^1.6.1",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"clean-webpack-plugin": "^3.0.0",
|
||||
"copy-webpack-plugin": "^6.0.3",
|
||||
"css-loader": "^3.4.2",
|
||||
"eslint": "^7.5.0",
|
||||
"eslint-config-airbnb-base": "^14.2.0",
|
||||
"css-loader": "5.2.7",
|
||||
"css-minimizer-webpack-plugin": "^4.0.0",
|
||||
"eslint": "^8.6.0",
|
||||
"eslint-config-airbnb": "^19.0.4",
|
||||
"eslint-config-airbnb-typescript": "^17.0.0",
|
||||
"eslint-plugin-html": "^6.0.2",
|
||||
"eslint-plugin-import": "^2.22.0",
|
||||
"eslint-plugin-import": "^2.25.3",
|
||||
"eslint-plugin-jsx-a11y": "^6.5.0",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"eslint-plugin-promise": "^4.2.1",
|
||||
"eslint-plugin-react": "^7.30.0",
|
||||
"eslint-plugin-react-hooks": "^4.5.0",
|
||||
"eslint-plugin-standard": "^4.0.1",
|
||||
"file-loader": "^6.0.0",
|
||||
"imports-loader": "^1.1.0",
|
||||
"mini-css-extract-plugin": "1.6.0",
|
||||
"jest": "^27.3.1",
|
||||
"mini-css-extract-plugin": "^1.6.2",
|
||||
"moment": "^2.29.3",
|
||||
"moment-locales-webpack-plugin": "^1.2.0",
|
||||
"optimize-css-assets-webpack-plugin": "6.0.0",
|
||||
"nock": "^13.2.4",
|
||||
"style-loader": "^1.2.1",
|
||||
"stylelint": "^13.6.1",
|
||||
"stylelint-config-standard": "^20.0.0",
|
||||
"terser-webpack-plugin": "<5.0.0",
|
||||
"typescript": "^4.6.3",
|
||||
"url-loader": "4.1.0",
|
||||
"webpack": "^4.16.3",
|
||||
"webpack-cli": "^3.1.0",
|
||||
"webpack-manifest-plugin": "^2.2.0"
|
||||
"webpack": "^5.73.0",
|
||||
"webpack-cli": "^4.0.0",
|
||||
"webpack-license-plugin": "^4.2.1",
|
||||
"webpack-manifest-plugin": "^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@chakra-ui/react": "^2.2.0",
|
||||
"@emotion/cache": "^11.9.3",
|
||||
"@emotion/react": "^11.9.3",
|
||||
"@emotion/styled": "^11",
|
||||
"axios": "^0.26.0",
|
||||
"bootstrap-3-typeahead": "^4.0.2",
|
||||
"camelcase-keys": "^7.0.0",
|
||||
"codemirror": "^5.59.1",
|
||||
"d3": "^3.4.4",
|
||||
"d3-shape": "^2.1.0",
|
||||
"d3-tip": "^0.9.1",
|
||||
"dagre-d3": "^0.6.4",
|
||||
"datatables.net": "^1.10.23",
|
||||
"datatables.net-bs": "^1.10.23",
|
||||
"datatables.net": "^1.11.4",
|
||||
"datatables.net-bs": "^1.11.4",
|
||||
"eonasdan-bootstrap-datetimepicker": "^4.17.47",
|
||||
"jquery": ">=3.4.0",
|
||||
"jshint": "^2.12.0",
|
||||
"moment-timezone": "^0.5.28",
|
||||
"framer-motion": "^6.0.0",
|
||||
"jquery": ">=3.5.0",
|
||||
"jshint": "^2.13.4",
|
||||
"lodash": "^4.17.21",
|
||||
"moment-timezone": "^0.5.34",
|
||||
"nvd3": "^1.8.6",
|
||||
"redoc": "^2.0.0-rc.48",
|
||||
"react": "^18.0.0",
|
||||
"react-dom": "^18.0.0",
|
||||
"react-icons": "^4.3.1",
|
||||
"react-query": "^3.39.1",
|
||||
"react-router-dom": "^6.3.0",
|
||||
"react-table": "^7.8.0",
|
||||
"redoc": "^2.0.0-rc.72",
|
||||
"url-search-params-polyfill": "^8.1.0"
|
||||
},
|
||||
"resolutions": {
|
||||
"lodash": "^4.17.21"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
303
pkgs/development/python-modules/apache-airflow/providers.nix
Normal file
303
pkgs/development/python-modules/apache-airflow/providers.nix
Normal file
File diff suppressed because one or more lines are too long
226
pkgs/development/python-modules/apache-airflow/update-providers.py
Executable file
226
pkgs/development/python-modules/apache-airflow/update-providers.py
Executable file
|
|
@ -0,0 +1,226 @@
|
|||
#! /usr/bin/env python3
|
||||
|
||||
from itertools import chain
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Set, TextIO
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import HTTPError
|
||||
import yaml
|
||||
|
||||
PKG_SET = "pkgs.python3Packages"
|
||||
|
||||
# If some requirements are matched by multiple or no Python packages, the
|
||||
# following can be used to choose the correct one
|
||||
PKG_PREFERENCES = {
|
||||
"dnspython": "dnspython",
|
||||
"google-api-python-client": "google-api-python-client",
|
||||
"psycopg2-binary": "psycopg2",
|
||||
"requests_toolbelt": "requests-toolbelt",
|
||||
}
|
||||
|
||||
# Requirements missing from the airflow provider metadata
|
||||
EXTRA_REQS = {
|
||||
"sftp": ["pysftp"],
|
||||
}
|
||||
|
||||
|
||||
def get_version():
|
||||
with open(os.path.dirname(sys.argv[0]) + "/default.nix") as fh:
|
||||
# A version consists of digits, dots, and possibly a "b" (for beta)
|
||||
m = re.search('version = "([\\d\\.b]+)";', fh.read())
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def get_file_from_github(version: str, path: str):
|
||||
with urlopen(
|
||||
f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}"
|
||||
) as response:
|
||||
return yaml.safe_load(response)
|
||||
|
||||
|
||||
def repository_root() -> Path:
|
||||
return Path(os.path.dirname(sys.argv[0])) / "../../../.."
|
||||
|
||||
|
||||
def dump_packages() -> Dict[str, Dict[str, str]]:
|
||||
# Store a JSON dump of Nixpkgs' python3Packages
|
||||
output = subprocess.check_output(
|
||||
[
|
||||
"nix-env",
|
||||
"-f",
|
||||
repository_root(),
|
||||
"-qa",
|
||||
"-A",
|
||||
PKG_SET,
|
||||
"--arg",
|
||||
"config",
|
||||
"{ allowAliases = false; }",
|
||||
"--json",
|
||||
]
|
||||
)
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def remove_version_constraint(req: str) -> str:
|
||||
return re.sub(r"[=><~].*$", "", req)
|
||||
|
||||
|
||||
def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
|
||||
if req in PKG_PREFERENCES:
|
||||
return f"{PKG_SET}.{PKG_PREFERENCES[req]}"
|
||||
attr_paths = []
|
||||
names = [req]
|
||||
# E.g. python-mpd2 is actually called python3.6-mpd2
|
||||
# instead of python-3.6-python-mpd2 inside Nixpkgs
|
||||
if req.startswith("python-") or req.startswith("python_"):
|
||||
names.append(req[len("python-") :])
|
||||
for name in names:
|
||||
# treat "-" and "_" equally
|
||||
name = re.sub("[-_]", "[-_]", name)
|
||||
# python(minor).(major)-(pname)-(version or unstable-date)
|
||||
# we need the version qualifier, or we'll have multiple matches
|
||||
# (e.g. pyserial and pyserial-asyncio when looking for pyserial)
|
||||
pattern = re.compile(
|
||||
f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I
|
||||
)
|
||||
for attr_path, package in packages.items():
|
||||
# logging.debug("Checking match for %s with %s", name, package["name"])
|
||||
if pattern.match(package["name"]):
|
||||
attr_paths.append(attr_path)
|
||||
# Let's hope there's only one derivation with a matching name
|
||||
assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}"
|
||||
if attr_paths:
|
||||
return attr_paths[0]
|
||||
return None
|
||||
|
||||
|
||||
def provider_reqs_to_attr_paths(reqs: List, packages: Dict) -> List:
|
||||
no_version_reqs = map(remove_version_constraint, reqs)
|
||||
filtered_reqs = [
|
||||
req for req in no_version_reqs if not re.match(r"^apache-airflow", req)
|
||||
]
|
||||
attr_paths = []
|
||||
for req in filtered_reqs:
|
||||
attr_path = name_to_attr_path(req, packages)
|
||||
if attr_path is not None:
|
||||
# Add attribute path without "python3Packages." prefix
|
||||
pname = attr_path[len(PKG_SET + ".") :]
|
||||
attr_paths.append(pname)
|
||||
else:
|
||||
# If we can't find it, we just skip and warn the user
|
||||
logging.warning("Could not find package attr for %s", req)
|
||||
return attr_paths
|
||||
|
||||
|
||||
def get_cross_provider_reqs(
|
||||
provider: str, provider_reqs: Dict, cross_provider_deps: Dict, seen: List = None
|
||||
) -> Set:
|
||||
# Unfortunately there are circular cross-provider dependencies, so keep a
|
||||
# list of ones we've seen already
|
||||
seen = seen or []
|
||||
reqs = set(provider_reqs[provider])
|
||||
if len(cross_provider_deps[provider]) > 0:
|
||||
reqs.update(
|
||||
chain.from_iterable(
|
||||
get_cross_provider_reqs(
|
||||
d, provider_reqs, cross_provider_deps, seen + [provider]
|
||||
)
|
||||
if d not in seen
|
||||
else []
|
||||
for d in cross_provider_deps[provider]
|
||||
)
|
||||
)
|
||||
return reqs
|
||||
|
||||
|
||||
def get_provider_reqs(version: str, packages: Dict) -> Dict:
|
||||
provider_dependencies = get_file_from_github(
|
||||
version, "generated/provider_dependencies.json"
|
||||
)
|
||||
provider_reqs = {}
|
||||
cross_provider_deps = {}
|
||||
for provider, provider_data in provider_dependencies.items():
|
||||
provider_reqs[provider] = list(
|
||||
provider_reqs_to_attr_paths(provider_data["deps"], packages)
|
||||
) + EXTRA_REQS.get(provider, [])
|
||||
cross_provider_deps[provider] = [
|
||||
d for d in provider_data["cross-providers-deps"] if d != "common.sql"
|
||||
]
|
||||
transitive_provider_reqs = {}
|
||||
# Add transitive cross-provider reqs
|
||||
for provider in provider_reqs:
|
||||
transitive_provider_reqs[provider] = get_cross_provider_reqs(
|
||||
provider, provider_reqs, cross_provider_deps
|
||||
)
|
||||
return transitive_provider_reqs
|
||||
|
||||
|
||||
def get_provider_yaml(version: str, provider: str) -> Dict:
|
||||
provider_dir = provider.replace(".", "/")
|
||||
path = f"airflow/providers/{provider_dir}/provider.yaml"
|
||||
try:
|
||||
return get_file_from_github(version, path)
|
||||
except HTTPError:
|
||||
logging.warning("Couldn't get provider yaml for %s", provider)
|
||||
return {}
|
||||
|
||||
|
||||
def get_provider_imports(version: str, providers) -> Dict:
|
||||
provider_imports = {}
|
||||
for provider in providers:
|
||||
provider_yaml = get_provider_yaml(version, provider)
|
||||
imports: List[str] = []
|
||||
if "hooks" in provider_yaml:
|
||||
imports.extend(
|
||||
chain.from_iterable(
|
||||
hook["python-modules"] for hook in provider_yaml["hooks"]
|
||||
)
|
||||
)
|
||||
if "operators" in provider_yaml:
|
||||
imports.extend(
|
||||
chain.from_iterable(
|
||||
operator["python-modules"]
|
||||
for operator in provider_yaml["operators"]
|
||||
)
|
||||
)
|
||||
provider_imports[provider] = imports
|
||||
return provider_imports
|
||||
|
||||
|
||||
def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None:
|
||||
fh.write("# Warning: generated by update-providers.py, do not update manually\n")
|
||||
fh.write("{\n")
|
||||
for provider, reqs in provider_reqs.items():
|
||||
provider_name = provider.replace(".", "_")
|
||||
fh.write(f" {provider_name} = {{\n")
|
||||
fh.write(
|
||||
" deps = [ " + " ".join(sorted(f'"{req}"' for req in reqs)) + " ];\n"
|
||||
)
|
||||
fh.write(
|
||||
" imports = [ "
|
||||
+ " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider]))
|
||||
+ " ];\n"
|
||||
)
|
||||
fh.write(" };\n")
|
||||
fh.write("}\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
version = get_version()
|
||||
packages = dump_packages()
|
||||
logging.info("Generating providers.nix for version %s", version)
|
||||
provider_reqs = get_provider_reqs(version, packages)
|
||||
provider_imports = get_provider_imports(version, provider_reqs.keys())
|
||||
with open("providers.nix", "w") as fh:
|
||||
to_nix_expr(provider_reqs, provider_imports, fh)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue