maintainers/scripts: add kde2nix tooling

This will be used to generate data for KDE 6 packages.
This commit is contained in:
K900 2024-02-05 18:43:08 +03:00
parent 2248bdfbbc
commit 3a52750ac7
6 changed files with 505 additions and 0 deletions

View file

@ -0,0 +1,31 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p gnutar jq reuse
set -eu
cd "$(dirname "$(readlink -f "$0")")"/../../..
TMPDIR=$(mktemp -d)
trap 'rm -rf $TMPDIR' EXIT
echo "# Prebuilding sources..."
nix-build -A kdePackages.sources --no-link || true
echo "# Evaluating sources..."
declare -A sources
eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)"
echo "# Collecting licenses..."
for k in "${!sources[@]}"; do
echo "- Processing $k..."
if [ ! -f "${sources[$k]}" ]; then
echo "Not found!"
continue
fi
mkdir "$TMPDIR/$k"
tar -C "$TMPDIR/$k" -xf "${sources[$k]}"
(cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json"
done
jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json

View file

@ -0,0 +1,13 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p gnused jq
set -eu
cd "$(dirname "$(readlink -f "$0")")"/../../..
mkdir -p logs
for name in $(nix-env -qaP -f . -A kdePackages --json | jq -r 'to_entries[] | .key' | sed s/kdePackages.//); do
echo "Processing ${name}..."
path=$(nix eval ".#kdePackages.${name}.outPath" --json --option warn-dirty false | jq -r)
if [ -n "${path}" ]; then
nix-store --read-log "${path}" > "logs/${name}.log" || true
fi
done

View file

@ -0,0 +1,36 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])"
import pathlib
import click
import utils
@click.command
@click.argument(
"repo-metadata",
type=click.Path(
exists=True,
file_okay=False,
resolve_path=True,
path_type=pathlib.Path,
),
)
@click.option(
"--nixpkgs",
type=click.Path(
exists=True,
file_okay=False,
resolve_path=True,
writable=True,
path_type=pathlib.Path,
),
default=pathlib.Path(__file__).parent.parent.parent.parent
)
def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path):
metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata)
out_dir = nixpkgs / "pkgs/kde/generated"
metadata.write_json(out_dir)
if __name__ == "__main__":
main() # type: ignore

View file

@ -0,0 +1,127 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p python3
import pathlib
OK_MISSING = {
# we don't use precompiled QML
'Qt6QuickCompiler',
'Qt6QmlCompilerPlusPrivate',
# usually used for version numbers
'Git',
# useless by itself, will warn if something else is not found
'PkgConfig',
# license verification
'ReuseTool',
# dev only
'ClangFormat',
# doesn't exist
'Qt6X11Extras',
}
OK_MISSING_BY_PACKAGE = {
"angelfish": {
"Qt6Feedback", # we don't have it
},
"attica": {
"Python3", # only used for license checks
},
"discover": {
"rpm-ostree-1", # we don't have rpm-ostree (duh)
"Snapd", # we don't have snaps and probably never will
},
"elisa": {
"UPNPQT", # upstream says it's broken
},
"extra-cmake-modules": {
"Sphinx", # only used for docs, bloats closure size
"QCollectionGenerator"
},
"kio-extras-kf5": {
"KDSoapWSDiscoveryClient", # actually vendored on KF5 version
},
"kitinerary": {
"OsmTools", # used for map data updates, we use prebuilt
},
"kosmindoormap": {
"OsmTools", # same
"Protobuf",
},
"kpty": {
"UTEMPTER", # we don't have it and it probably wouldn't work anyway
},
"kpublictransport": {
"OsmTools", # same
"PolyClipping",
"Protobuf",
},
"krfb": {
"Qt6XkbCommonSupport", # not real
},
"kuserfeedback": {
"Qt6Svg", # all used for backend console stuff we don't ship
"QmlLint",
"Qt6Charts",
"FLEX",
"BISON",
"Php",
"PhpUnit",
},
"kwin": {
"display-info", # newer versions identify as libdisplay-info
},
"mlt": {
"Qt5", # intentionally disabled
"SWIG",
},
"plasma-desktop": {
"scim", # upstream is dead, not packaged in Nixpkgs
},
"powerdevil": {
"DDCUtil", # cursed, intentionally disabled
},
"pulseaudio-qt": {
"Qt6Qml", # tests only
"Qt6Quick",
},
"syntax-highlighting": {
"XercesC", # only used for extra validation at build time
}
}
def main():
here = pathlib.Path(__file__).parent.parent.parent.parent
logs = (here / "logs").glob("*.log")
for log in sorted(logs):
pname = log.stem
missing = []
is_in_block = False
with log.open(errors="replace") as fd:
for line in fd:
line = line.strip()
if line.startswith("-- No package '"):
package = line.removeprefix("-- No package '").removesuffix("' found")
missing.append(package)
if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:":
is_in_block = True
elif line.startswith("--") and is_in_block:
is_in_block = False
elif line.startswith("*") and is_in_block:
package = line.removeprefix("* ")
missing.append(package)
missing = {
package
for package in missing
if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set()))
}
if missing:
print(pname + ":")
for line in missing:
print(" -", line)
print()
if __name__ == '__main__':
main()

View file

@ -0,0 +1,113 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.pyyaml ])
import base64
import binascii
import json
import pathlib
from urllib.parse import urlparse
import bs4
import click
import httpx
import jinja2
import utils
LEAF_TEMPLATE = jinja2.Template('''
{mkKdeDerivation}:
mkKdeDerivation {
pname = "{{ pname }}";
}
'''.strip())
ROOT_TEMPLATE = jinja2.Template('''
{callPackage}: {
{%- for p in packages %}
{{ p }} = callPackage ./{{ p }} {};
{%- endfor %}
}
'''.strip());
def to_sri(hash):
raw = binascii.unhexlify(hash)
b64 = base64.b64encode(raw).decode()
return f"sha256-{b64}"
@click.command
@click.argument(
"set",
type=click.Choice(["frameworks", "gear", "plasma"]),
required=True
)
@click.argument(
"version",
type=str,
required=True
)
@click.option(
"--nixpkgs",
type=click.Path(
exists=True,
file_okay=False,
resolve_path=True,
writable=True,
path_type=pathlib.Path,
),
default=pathlib.Path(__file__).parent.parent.parent.parent
)
def main(set: str, version: str, nixpkgs: pathlib.Path):
root_dir = nixpkgs / "pkgs/kde"
set_dir = root_dir / set
generated_dir = root_dir / "generated"
metadata = utils.KDERepoMetadata.from_json(generated_dir)
set_url = {
"frameworks": "kf",
"gear": "releases",
"plasma": "plasma",
}[set]
sources = httpx.get(f"https://kde.org/info/sources/source-{set_url}-{version}.html")
sources.raise_for_status()
bs = bs4.BeautifulSoup(sources.text, features="html.parser")
results = {}
for item in bs.select("tr")[1:]:
link = item.select_one("td:nth-child(1) a")
assert link
hash = item.select_one("td:nth-child(3) tt")
assert hash
project_name, version = link.text.rsplit("-", maxsplit=1)
if project_name not in metadata.projects_by_name:
print(f"Warning: unknown tarball: {project_name}")
results[project_name] = {
"version": version,
"url": "mirror://kde" + urlparse(link.attrs["href"]).path,
"hash": to_sri(hash.text)
}
pkg_dir = set_dir / project_name
pkg_file = pkg_dir / "default.nix"
if not pkg_file.exists():
print(f"Generated new package: {set}/{project_name}")
pkg_dir.mkdir(parents=True, exist_ok=True)
with pkg_file.open("w") as fd:
fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
set_dir.mkdir(parents=True, exist_ok=True)
with (set_dir / "default.nix").open("w") as fd:
fd.write(ROOT_TEMPLATE.render(packages=results.keys()) + "\n")
sources_dir = generated_dir / "sources"
sources_dir.mkdir(parents=True, exist_ok=True)
with (sources_dir / f"{set}.json").open("w") as fd:
json.dump(results, fd, indent=2)
if __name__ == "__main__":
main() # type: ignore

View file

@ -0,0 +1,185 @@
import collections
import dataclasses
import functools
import json
import pathlib
import subprocess
import yaml
class DataclassEncoder(json.JSONEncoder):
def default(self, it):
if dataclasses.is_dataclass(it):
return dataclasses.asdict(it)
return super().default(it)
@dataclasses.dataclass
class Project:
name: str
description: str | None
project_path: str
repo_path: str | None
def __hash__(self) -> int:
return hash(self.name)
@classmethod
def from_yaml(cls, path: pathlib.Path):
data = yaml.safe_load(path.open())
return cls(
name=data["identifier"],
description=data["description"],
project_path=data["projectpath"],
repo_path=data["repopath"]
)
def get_git_commit(path: pathlib.Path):
return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip()
def validate_unique(projects: list[Project], attr: str):
seen = set()
for item in projects:
attr_value = getattr(item, attr)
if attr_value in seen:
raise Exception(f"Duplicate {attr}: {attr_value}")
seen.add(attr_value)
THIRD_PARTY = {
"third-party/appstream": "appstream-qt",
"third-party/cmark": "cmark",
"third-party/gpgme": "gpgme",
"third-party/kdsoap": "kdsoap",
"third-party/libaccounts-qt": "accounts-qt",
"third-party/libgpg-error": "libgpg-error",
"third-party/libquotient": "libquotient",
"third-party/packagekit-qt": "packagekit-qt",
"third-party/poppler": "poppler",
"third-party/qcoro": "qcoro",
"third-party/qmltermwidget": "qmltermwidget",
"third-party/qtkeychain": "qtkeychain",
"third-party/signond": "signond",
"third-party/taglib": "taglib",
"third-party/wayland-protocols": "wayland-protocols",
"third-party/wayland": "wayland",
"third-party/zxing-cpp": "zxing-cpp",
}
IGNORE = {
"kdesupport/phonon-directshow",
"kdesupport/phonon-mmf",
"kdesupport/phonon-mplayer",
"kdesupport/phonon-quicktime",
"kdesupport/phonon-waveout",
"kdesupport/phonon-xine"
}
WARNED = set()
@dataclasses.dataclass
class KDERepoMetadata:
version: str
projects: list[Project]
dep_graph: dict[Project, set[Project]]
@functools.cached_property
def projects_by_name(self):
return {p.name: p for p in self.projects}
@functools.cached_property
def projects_by_path(self):
return {p.project_path: p for p in self.projects}
def try_lookup_package(self, path):
if path in IGNORE:
return None
project = self.projects_by_path.get(path)
if project is None and path not in WARNED:
WARNED.add(path)
print(f"Warning: unknown project {path}")
return project
@classmethod
def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path):
projects = [
Project.from_yaml(metadata_file)
for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml")
] + [
Project(id, None, project_path, None)
for project_path, id in THIRD_PARTY.items()
]
validate_unique(projects, "name")
validate_unique(projects, "project_path")
self = cls(
version=get_git_commit(repo_metadata),
projects=projects,
dep_graph={},
)
dep_specs = [
"dependency-data-common",
"dependency-data-kf6-qt6"
]
dep_graph = collections.defaultdict(set)
for spec in dep_specs:
spec_path = repo_metadata / "dependencies" / spec
for line in spec_path.open():
line = line.strip()
if line.startswith("#"):
continue
if not line:
continue
dependent, dependency = line.split(": ")
dependent = self.try_lookup_package(dependent)
if dependent is None:
continue
dependency = self.try_lookup_package(dependency)
if dependency is None:
continue
dep_graph[dependent].add(dependency)
self.dep_graph = dep_graph
return self
def write_json(self, root: pathlib.Path):
root.mkdir(parents=True, exist_ok=True)
with (root / "projects.json").open("w") as fd:
json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
with (root / "dependencies.json").open("w") as fd:
deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()}
json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
@classmethod
def from_json(cls, root: pathlib.Path):
projects = [
Project(**v) for v in json.load((root / "projects.json").open()).values()
]
deps = json.load((root / "dependencies.json").open())
self = cls(
version=deps["version"],
projects=projects,
dep_graph={},
)
dep_graph = collections.defaultdict(set)
for dependent, dependencies in deps["dependencies"].items():
for dependency in dependencies:
dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency])
self.dep_graph = dep_graph
return self