From 1229e735ac51dbe79724f7648655a2089c9c67b9 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 11 Feb 2023 08:48:06 +0100 Subject: [PATCH] nixos-render-docs: add structural includes, use for manual this adds support for structural includes to nixos-render-docs. structural includes provide a way to denote the (sub)structure of the nixos manual in the markdown source files, very similar to how we used literal docbook blocks before, and are processed by nixos-render-docs without involvement of xml tooling. this will ultimately allow us to emit the nixos manual in other formats as well, e.g. html, without going through docbook at all. alternatives to this source layout were also considered: a parallel structure using e.g. toml files that describe the document tree and links to each part is possible, but much more complicated to implement than the solution chosen here and makes it harder to follow which files have what substructure. it also makes it much harder to include a substructure in the middle of a file. much the same goes for command-line arguments to the converter, only that command-lined arguments are even harder to specify correctly and cannot be reasonably pulled together from many places without involving another layer of tooling. cli arguments would also mean that the manual structure would be fixed in default.nix, which is also not ideal. --- .../administration/containers.chapter.md | 8 +- nixos/doc/manual/administration/running.md | 14 + nixos/doc/manual/administration/running.xml | 21 -- .../administration/troubleshooting.chapter.md | 12 +- .../configuration/config-syntax.chapter.md | 8 +- .../doc/manual/configuration/configuration.md | 27 ++ .../manual/configuration/configuration.xml | 31 -- .../declarative-packages.section.md | 6 +- .../configuration/file-systems.chapter.md | 6 +- .../configuration/networking.chapter.md | 18 +- .../configuration/package-mgmt.chapter.md | 6 +- .../manual/configuration/profiles.chapter.md | 24 +- nixos/doc/manual/default.nix | 45 +-- nixos/doc/manual/development/development.md | 14 + nixos/doc/manual/development/development.xml | 20 -- .../manual/development/nixos-tests.chapter.md | 10 +- ...-happens-during-a-system-switch.chapter.md | 6 +- .../writing-documentation.chapter.md | 2 +- .../development/writing-modules.chapter.md | 20 +- .../writing-nixos-tests.section.md | 6 +- nixos/doc/manual/installation/installation.md | 11 + .../doc/manual/installation/installation.xml | 18 - .../manual/installation/installing.chapter.md | 14 +- nixos/doc/manual/manual.md | 53 +++ nixos/doc/manual/manual.xml | 23 -- nixos/doc/manual/nixos-options.md | 7 + nixos/doc/manual/preface.md | 11 + nixos/doc/manual/preface.xml | 42 --- .../doc/manual/release-notes/release-notes.md | 25 ++ .../manual/release-notes/release-notes.xml | 30 -- .../src/nixos_render_docs/manual.py | 315 ++++++++++-------- .../src/nixos_render_docs/options.py | 5 +- 32 files changed, 427 insertions(+), 431 deletions(-) create mode 100644 nixos/doc/manual/administration/running.md delete mode 100644 nixos/doc/manual/administration/running.xml create mode 100644 nixos/doc/manual/configuration/configuration.md delete mode 100644 nixos/doc/manual/configuration/configuration.xml create mode 100644 nixos/doc/manual/development/development.md delete mode 100644 nixos/doc/manual/development/development.xml create mode 100644 nixos/doc/manual/installation/installation.md delete mode 100644 nixos/doc/manual/installation/installation.xml create mode 100644 nixos/doc/manual/manual.md delete mode 100644 nixos/doc/manual/manual.xml create mode 100644 nixos/doc/manual/nixos-options.md create mode 100644 nixos/doc/manual/preface.md delete mode 100644 nixos/doc/manual/preface.xml create mode 100644 nixos/doc/manual/release-notes/release-notes.md delete mode 100644 nixos/doc/manual/release-notes/release-notes.xml diff --git a/nixos/doc/manual/administration/containers.chapter.md b/nixos/doc/manual/administration/containers.chapter.md index ea51f91f698f..50493b562b54 100644 --- a/nixos/doc/manual/administration/containers.chapter.md +++ b/nixos/doc/manual/administration/containers.chapter.md @@ -21,8 +21,8 @@ which is often not what you want. By contrast, in the imperative approach, containers are configured and updated independently from the host system. -```{=docbook} - - - +```{=include=} sections +imperative-containers.section.md +declarative-containers.section.md +container-networking.section.md ``` diff --git a/nixos/doc/manual/administration/running.md b/nixos/doc/manual/administration/running.md new file mode 100644 index 000000000000..48e8c7c6668b --- /dev/null +++ b/nixos/doc/manual/administration/running.md @@ -0,0 +1,14 @@ +# Administration {#ch-running} + +This chapter describes various aspects of managing a running NixOS system, such as how to use the {command}`systemd` service manager. + +```{=include=} chapters +service-mgmt.chapter.md +rebooting.chapter.md +user-sessions.chapter.md +control-groups.chapter.md +logging.chapter.md +cleaning-store.chapter.md +containers.chapter.md +troubleshooting.chapter.md +``` diff --git a/nixos/doc/manual/administration/running.xml b/nixos/doc/manual/administration/running.xml deleted file mode 100644 index d9fcc1aee263..000000000000 --- a/nixos/doc/manual/administration/running.xml +++ /dev/null @@ -1,21 +0,0 @@ - - Administration - - - This chapter describes various aspects of managing a running NixOS system, - such as how to use the systemd service manager. - - - - - - - - - - - diff --git a/nixos/doc/manual/administration/troubleshooting.chapter.md b/nixos/doc/manual/administration/troubleshooting.chapter.md index 548456eaf6d6..1253607f8efc 100644 --- a/nixos/doc/manual/administration/troubleshooting.chapter.md +++ b/nixos/doc/manual/administration/troubleshooting.chapter.md @@ -3,10 +3,10 @@ This chapter describes solutions to common problems you might encounter when you manage your NixOS system. -```{=docbook} - - - - - +```{=include=} sections +boot-problems.section.md +maintenance-mode.section.md +rollback.section.md +store-corruption.section.md +network-problems.section.md ``` diff --git a/nixos/doc/manual/configuration/config-syntax.chapter.md b/nixos/doc/manual/configuration/config-syntax.chapter.md index 9f8d45d58899..9e606b2b82af 100644 --- a/nixos/doc/manual/configuration/config-syntax.chapter.md +++ b/nixos/doc/manual/configuration/config-syntax.chapter.md @@ -11,8 +11,8 @@ manual](https://nixos.org/nix/manual/#chap-writing-nix-expressions), but here we give a short overview of the most important constructs useful in NixOS configuration files. -```{=docbook} - - - +```{=include=} sections +config-file.section.md +abstractions.section.md +modularity.section.md ``` diff --git a/nixos/doc/manual/configuration/configuration.md b/nixos/doc/manual/configuration/configuration.md new file mode 100644 index 000000000000..4c966f3325b9 --- /dev/null +++ b/nixos/doc/manual/configuration/configuration.md @@ -0,0 +1,27 @@ +# Configuration {#ch-configuration} + +This chapter describes how to configure various aspects of a NixOS machine through the configuration file {file}`/etc/nixos/configuration.nix`. As described in [](#sec-changing-config), changes to this file only take effect after you run {command}`nixos-rebuild`. + +```{=include=} chapters +config-syntax.chapter.md +package-mgmt.chapter.md +user-mgmt.chapter.md +file-systems.chapter.md +x-windows.chapter.md +wayland.chapter.md +gpu-accel.chapter.md +xfce.chapter.md +networking.chapter.md +linux-kernel.chapter.md +subversion.chapter.md +``` + +```{=include=} chapters +@MODULE_CHAPTERS@ +``` + +```{=include=} chapters +profiles.chapter.md +kubernetes.chapter.md +``` + diff --git a/nixos/doc/manual/configuration/configuration.xml b/nixos/doc/manual/configuration/configuration.xml deleted file mode 100644 index b04316cfa48e..000000000000 --- a/nixos/doc/manual/configuration/configuration.xml +++ /dev/null @@ -1,31 +0,0 @@ - - Configuration - - - This chapter describes how to configure various aspects of a NixOS machine - through the configuration file - /etc/nixos/configuration.nix. As described in - , changes to this file only take - effect after you run nixos-rebuild. - - - - - - - - - - - - - - - - - - diff --git a/nixos/doc/manual/configuration/declarative-packages.section.md b/nixos/doc/manual/configuration/declarative-packages.section.md index 337cdf8472e4..02eaa56192e4 100644 --- a/nixos/doc/manual/configuration/declarative-packages.section.md +++ b/nixos/doc/manual/configuration/declarative-packages.section.md @@ -40,7 +40,7 @@ configuration use `pkgs` prefix (variable). To "uninstall" a package, simply remove it from [](#opt-environment.systemPackages) and run `nixos-rebuild switch`. -```{=docbook} - - +```{=include=} sections +customizing-packages.section.md +adding-custom-packages.section.md ``` diff --git a/nixos/doc/manual/configuration/file-systems.chapter.md b/nixos/doc/manual/configuration/file-systems.chapter.md index 901e2e4f181b..aca978be064d 100644 --- a/nixos/doc/manual/configuration/file-systems.chapter.md +++ b/nixos/doc/manual/configuration/file-systems.chapter.md @@ -36,7 +36,7 @@ dropping you to the emergency shell. You can make a mount asynchronous and non-critical by adding `options = [ "nofail" ];`. ::: -```{=docbook} - - +```{=include=} sections +luks-file-systems.section.md +sshfs-file-systems.section.md ``` diff --git a/nixos/doc/manual/configuration/networking.chapter.md b/nixos/doc/manual/configuration/networking.chapter.md index 529dc0610bda..abbd9766f173 100644 --- a/nixos/doc/manual/configuration/networking.chapter.md +++ b/nixos/doc/manual/configuration/networking.chapter.md @@ -3,14 +3,14 @@ This section describes how to configure networking components on your NixOS machine. -```{=docbook} - - - - - - - - +```{=include=} sections +network-manager.section.md +ssh.section.md +ipv4-config.section.md +ipv6-config.section.md +firewall.section.md +wireless.section.md +ad-hoc-network-config.section.md +renaming-interfaces.section.md ``` diff --git a/nixos/doc/manual/configuration/package-mgmt.chapter.md b/nixos/doc/manual/configuration/package-mgmt.chapter.md index a6c414be59a9..1148bbe84740 100644 --- a/nixos/doc/manual/configuration/package-mgmt.chapter.md +++ b/nixos/doc/manual/configuration/package-mgmt.chapter.md @@ -12,7 +12,7 @@ NixOS has two distinct styles of package management: `nix-env` command. This style allows mixing packages from different Nixpkgs versions. It's the only choice for non-root users. -```{=docbook} - - +```{=include=} sections +declarative-packages.section.md +ad-hoc-packages.section.md ``` diff --git a/nixos/doc/manual/configuration/profiles.chapter.md b/nixos/doc/manual/configuration/profiles.chapter.md index 2c3dea27c181..9f1f48f742ac 100644 --- a/nixos/doc/manual/configuration/profiles.chapter.md +++ b/nixos/doc/manual/configuration/profiles.chapter.md @@ -19,16 +19,16 @@ install media, many are actually intended to be used in real installs. What follows is a brief explanation on the purpose and use-case for each profile. Detailing each option configured by each one is out of scope. -```{=docbook} - - - - - - - - - - - +```{=include=} sections +profiles/all-hardware.section.md +profiles/base.section.md +profiles/clone-config.section.md +profiles/demo.section.md +profiles/docker-container.section.md +profiles/graphical.section.md +profiles/hardened.section.md +profiles/headless.section.md +profiles/installation-device.section.md +profiles/minimal.section.md +profiles/qemu-guest.section.md ``` diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 3da6e88c70b7..8c71e5108799 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -143,35 +143,26 @@ let '' cp -r --no-preserve=all $inputs/* . - declare -a convert_args - while read -r mf; do - if [[ "$mf" = *.chapter.md ]]; then - convert_args+=("--chapter") - else - convert_args+=("--section") - fi + substituteInPlace ./manual.md \ + --replace '@NIXOS_VERSION@' "${version}" + substituteInPlace ./configuration/configuration.md \ + --replace \ + '@MODULE_CHAPTERS@' \ + ${lib.escapeShellArg (lib.concatMapStringsSep "\n" (p: "${p.value}") config.meta.doc)} + substituteInPlace ./nixos-options.md \ + --replace \ + '@NIXOS_OPTIONS_JSON@' \ + ${optionsDoc.optionsJSON}/share/doc/nixos/options.json + substituteInPlace ./development/writing-nixos-tests.section.md \ + --replace \ + '@NIXOS_TEST_OPTIONS_JSON@' \ + ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json - convert_args+=("from_md/''${mf%.md}.xml" "$mf") - done < <(find . -type f -name '*.md') - - nixos-render-docs manual docbook-fragment \ + nixos-render-docs manual docbook \ --manpage-urls ${manpageUrls} \ - "''${convert_args[@]}" - - mkdir ./generated - ln -s ${optionsDoc.optionsDocBook} ./generated/options-db.xml - ln -s ${testOptionsDoc.optionsDocBook} ./generated/test-options-db.xml - printf "%s" "${version}" > ./generated/version - chmod -R u+w . - - nixos-render-docs manual docbook-section \ - --manpage-urls ${manpageUrls} \ - ./generated/modules.xml \ - --section \ - --section-id modules \ - --chapters ${lib.concatMapStrings (p: "${p.value} ") config.meta.doc} - - xmllint --xinclude --output ./manual-combined.xml ./manual.xml + --revision ${lib.escapeShellArg revision} \ + ./manual.md \ + ./manual-combined.xml ${linterFunctions} diff --git a/nixos/doc/manual/development/development.md b/nixos/doc/manual/development/development.md new file mode 100644 index 000000000000..6a0dd091b129 --- /dev/null +++ b/nixos/doc/manual/development/development.md @@ -0,0 +1,14 @@ +# Development {#ch-development} + +This chapter describes how you can modify and extend NixOS. + +```{=include=} chapters +sources.chapter.md +writing-modules.chapter.md +building-parts.chapter.md +bootspec.chapter.md +what-happens-during-a-system-switch.chapter.md +writing-documentation.chapter.md +nixos-tests.chapter.md +testing-installer.chapter.md +``` diff --git a/nixos/doc/manual/development/development.xml b/nixos/doc/manual/development/development.xml deleted file mode 100644 index 949468c9021d..000000000000 --- a/nixos/doc/manual/development/development.xml +++ /dev/null @@ -1,20 +0,0 @@ - - Development - - - This chapter describes how you can modify and extend NixOS. - - - - - - - - - - - diff --git a/nixos/doc/manual/development/nixos-tests.chapter.md b/nixos/doc/manual/development/nixos-tests.chapter.md index 2a4fdddeaa66..ec0e4b9f076a 100644 --- a/nixos/doc/manual/development/nixos-tests.chapter.md +++ b/nixos/doc/manual/development/nixos-tests.chapter.md @@ -5,9 +5,9 @@ NixOS tests are kept in the directory `nixos/tests`, and are executed (using Nix) by a testing framework that automatically starts one or more virtual machines containing the NixOS system(s) required for the test. -```{=docbook} - - - - +```{=include=} sections +writing-nixos-tests.section.md +running-nixos-tests.section.md +running-nixos-tests-interactively.section.md +linking-nixos-tests-to-packages.section.md ``` diff --git a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md index aad82831a3c2..9cbec729803a 100644 --- a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md +++ b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md @@ -47,7 +47,7 @@ Most of these actions are either self-explaining but some of them have to do with our units or the activation script. For this reason, these topics are explained in the next sections. -```{=docbook} - - +```{=include=} sections +unit-handling.section.md +activation-script.section.md ``` diff --git a/nixos/doc/manual/development/writing-documentation.chapter.md b/nixos/doc/manual/development/writing-documentation.chapter.md index 4986c9f0a81b..8d504dfb0b0a 100644 --- a/nixos/doc/manual/development/writing-documentation.chapter.md +++ b/nixos/doc/manual/development/writing-documentation.chapter.md @@ -83,7 +83,7 @@ Keep the following guidelines in mind when you create and add a topic: ## Adding a Topic to the Book {#sec-writing-docs-adding-a-topic} -Open the parent XML file and add an `xi:include` element to the list of +Open the parent CommonMark file and add a line to the list of chapters with the file name of the topic that you created. If you created a `section`, you add the file to the `chapter` file. If you created a `chapter`, you add the file to the `part` file. diff --git a/nixos/doc/manual/development/writing-modules.chapter.md b/nixos/doc/manual/development/writing-modules.chapter.md index a0ec4a5df96e..ae657458d768 100644 --- a/nixos/doc/manual/development/writing-modules.chapter.md +++ b/nixos/doc/manual/development/writing-modules.chapter.md @@ -189,14 +189,14 @@ in { ``` ::: -```{=docbook} - - - - - - - - - +```{=include=} sections +option-declarations.section.md +option-types.section.md +option-def.section.md +assertions.section.md +meta-attributes.section.md +importing-modules.section.md +replace-modules.section.md +freeform-modules.section.md +settings-options.section.md ``` diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md index d80e314e6251..3de46fda3df6 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.section.md +++ b/nixos/doc/manual/development/writing-nixos-tests.section.md @@ -470,6 +470,8 @@ In that case, `numpy` is chosen from the generic `python3Packages`. The following options can be used when writing tests. -```{=docbook} - +```{=include=} options +id-prefix: test-opt- +list-id: test-options-list +source: @NIXOS_TEST_OPTIONS_JSON@ ``` diff --git a/nixos/doc/manual/installation/installation.md b/nixos/doc/manual/installation/installation.md new file mode 100644 index 000000000000..140594256609 --- /dev/null +++ b/nixos/doc/manual/installation/installation.md @@ -0,0 +1,11 @@ +# Installation {#ch-installation} + +This section describes how to obtain, install, and configure NixOS for first-time use. + +```{=include=} chapters +obtaining.chapter.md +installing.chapter.md +changing-config.chapter.md +upgrading.chapter.md +building-nixos.chapter.md +``` diff --git a/nixos/doc/manual/installation/installation.xml b/nixos/doc/manual/installation/installation.xml deleted file mode 100644 index ba07d71d0ca3..000000000000 --- a/nixos/doc/manual/installation/installation.xml +++ /dev/null @@ -1,18 +0,0 @@ - - Installation - - - This section describes how to obtain, install, and configure NixOS for - first-time use. - - - - - - - - diff --git a/nixos/doc/manual/installation/installing.chapter.md b/nixos/doc/manual/installation/installing.chapter.md index e1908017a7e4..cf783c2d22b6 100644 --- a/nixos/doc/manual/installation/installing.chapter.md +++ b/nixos/doc/manual/installation/installing.chapter.md @@ -602,11 +602,11 @@ With a partitioned disk. ## Additional installation notes {#sec-installation-additional-notes} -```{=docbook} - - - - - - +```{=include=} sections +installing-usb.section.md +installing-pxe.section.md +installing-kexec.section.md +installing-virtualbox-guest.section.md +installing-from-other-distro.section.md +installing-behind-a-proxy.section.md ``` diff --git a/nixos/doc/manual/manual.md b/nixos/doc/manual/manual.md new file mode 100644 index 000000000000..1972eaeda872 --- /dev/null +++ b/nixos/doc/manual/manual.md @@ -0,0 +1,53 @@ +# NixOS Manual {#book-nixos-manual} +## Version @NIXOS_VERSION@ + + + +```{=include=} preface +preface.md +``` + +```{=include=} parts +installation/installation.md +configuration/configuration.md +administration/running.md +development/development.md +``` + +```{=include=} chapters +contributing-to-this-manual.chapter.md +``` + +```{=include=} appendix +nixos-options.md +release-notes/release-notes.md +``` diff --git a/nixos/doc/manual/manual.xml b/nixos/doc/manual/manual.xml deleted file mode 100644 index 4440f8e04baa..000000000000 --- a/nixos/doc/manual/manual.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - NixOS Manual - Version - - - - - - - - - - Configuration Options - - - - diff --git a/nixos/doc/manual/nixos-options.md b/nixos/doc/manual/nixos-options.md new file mode 100644 index 000000000000..33b487c95a2e --- /dev/null +++ b/nixos/doc/manual/nixos-options.md @@ -0,0 +1,7 @@ +# Configuration Options {#ch-options} + +```{=include=} options +id-prefix: opt- +list-id: configuration-variable-list +source: @NIXOS_OPTIONS_JSON@ +``` diff --git a/nixos/doc/manual/preface.md b/nixos/doc/manual/preface.md new file mode 100644 index 000000000000..d5e6364780a7 --- /dev/null +++ b/nixos/doc/manual/preface.md @@ -0,0 +1,11 @@ +# Preface {#preface} + +This manual describes how to install, use and extend NixOS, a Linux distribution based on the purely functional package management system [Nix](https://nixos.org/nix), that is composed using modules and packages defined in the [Nixpkgs](https://nixos.org/nixpkgs) project. + +Additional information regarding the Nix package manager and the Nixpkgs project can be found in respectively the [Nix manual](https://nixos.org/nix/manual) and the [Nixpkgs manual](https://nixos.org/nixpkgs/manual). + +If you encounter problems, please report them on the [`Discourse`](https://discourse.nixos.org), the [Matrix room](https://matrix.to/#nix:nixos.org), or on the [`#nixos` channel on Libera.Chat](irc://irc.libera.chat/#nixos). Alternatively, consider [contributing to this manual](#chap-contributing). Bugs should be reported in [NixOS’ GitHub issue tracker](https://github.com/NixOS/nixpkgs/issues). + +::: {.note} +Commands prefixed with `#` have to be run as root, either requiring to login as root user or temporarily switching to it using `sudo` for example. +::: diff --git a/nixos/doc/manual/preface.xml b/nixos/doc/manual/preface.xml deleted file mode 100644 index c0d530c3d1b5..000000000000 --- a/nixos/doc/manual/preface.xml +++ /dev/null @@ -1,42 +0,0 @@ - - Preface - - This manual describes how to install, use and extend NixOS, a Linux - distribution based on the purely functional package management system - Nix, that is composed - using modules and packages defined in the - Nixpkgs project. - - - Additional information regarding the Nix package manager and the Nixpkgs - project can be found in respectively the - Nix manual and the - Nixpkgs manual. - - - If you encounter problems, please report them on the - Discourse, - the Matrix room, - or on the - #nixos channel on Libera.Chat. - Alternatively, consider - contributing to this manual. Bugs should be - reported in - NixOS’ - GitHub issue tracker. - - - - Commands prefixed with # have to be run as root, either - requiring to login as root user or temporarily switching to it using - sudo for example. - - - diff --git a/nixos/doc/manual/release-notes/release-notes.md b/nixos/doc/manual/release-notes/release-notes.md new file mode 100644 index 000000000000..ac61de3793e8 --- /dev/null +++ b/nixos/doc/manual/release-notes/release-notes.md @@ -0,0 +1,25 @@ +# Release Notes {#ch-release-notes} + +This section lists the release notes for each stable version of NixOS and current unstable revision. + +```{=include=} sections +rl-2305.section.md +rl-2211.section.md +rl-2205.section.md +rl-2111.section.md +rl-2105.section.md +rl-2009.section.md +rl-2003.section.md +rl-1909.section.md +rl-1903.section.md +rl-1809.section.md +rl-1803.section.md +rl-1709.section.md +rl-1703.section.md +rl-1609.section.md +rl-1603.section.md +rl-1509.section.md +rl-1412.section.md +rl-1404.section.md +rl-1310.section.md +``` diff --git a/nixos/doc/manual/release-notes/release-notes.xml b/nixos/doc/manual/release-notes/release-notes.xml deleted file mode 100644 index bb5cc677afb8..000000000000 --- a/nixos/doc/manual/release-notes/release-notes.xml +++ /dev/null @@ -1,30 +0,0 @@ - - Release Notes - - This section lists the release notes for each stable version of NixOS and - current unstable revision. - - - - - - - - - - - - - - - - - - - - - diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index ab61d699d7f5..efc8b02e8d6b 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -2,68 +2,107 @@ import argparse import json from abc import abstractmethod -from collections.abc import MutableMapping, Sequence +from collections.abc import Mapping, MutableMapping, Sequence from pathlib import Path from typing import Any, cast, NamedTuple, Optional, Union from xml.sax.saxutils import escape, quoteattr + +import markdown_it from markdown_it.token import Token from markdown_it.utils import OptionsDict -from .docbook import DocBookRenderer +from . import options +from .docbook import DocBookRenderer, Heading from .md import Converter -class RenderedSection: - id: Optional[str] - chapters: list[str] - - def __init__(self, id: Optional[str]) -> None: - self.id = id - self.chapters = [] - -class BaseConverter(Converter): - _sections: list[RenderedSection] - - def __init__(self, manpage_urls: dict[str, str]): - super().__init__(manpage_urls) - self._sections = [] - - def add_section(self, id: Optional[str], chapters: list[Path]) -> None: - self._sections.append(RenderedSection(id)) - for chpath in chapters: - try: - with open(chpath, 'r') as f: - self._md.renderer._title_seen = False # type: ignore[attr-defined] - self._sections[-1].chapters.append(self._render(f.read())) - except Exception as e: - raise RuntimeError(f"failed to render manual chapter {chpath}") from e - - @abstractmethod - def finalize(self) -> str: raise NotImplementedError() - class ManualDocBookRenderer(DocBookRenderer): - # needed to check correctness of chapters. - # we may want to use front matter instead of this kind of heuristic. - _title_seen = False + _toplevel_tag: str + + def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str], + parser: Optional[markdown_it.MarkdownIt] = None): + super().__init__(manpage_urls, parser) + self._toplevel_tag = toplevel_tag + self.rules |= { + 'included_sections': lambda *args: self._included_thing("section", *args), + 'included_chapters': lambda *args: self._included_thing("chapter", *args), + 'included_preface': lambda *args: self._included_thing("preface", *args), + 'included_parts': lambda *args: self._included_thing("part", *args), + 'included_appendix': lambda *args: self._included_thing("appendix", *args), + 'included_options': self.included_options, + } + + def render(self, tokens: Sequence[Token], options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + wanted = { 'h1': 'title' } + wanted |= { 'h2': 'subtitle' } if self._toplevel_tag == 'book' else {} + for (i, (tag, kind)) in enumerate(wanted.items()): + if len(tokens) < 3 * (i + 1): + raise RuntimeError(f"missing {kind} ({tag}) heading") + token = tokens[3 * i] + if token.type != 'heading_open' or token.tag != tag: + assert token.map + raise RuntimeError(f"expected {kind} ({tag}) heading in line {token.map[0] + 1}", token) + for t in tokens[3 * len(wanted):]: + if t.type != 'heading_open' or (info := wanted.get(t.tag)) is None: + continue + assert t.map + raise RuntimeError( + f"only one {info[0]} heading ({t.markup} [text...]) allowed per " + f"{self._toplevel_tag}, but found a second in lines [{t.map[0] + 1}..{t.map[1]}]. " + "please remove all such headings except the first or demote the subsequent headings.", + t) + + # books get special handling because they have *two* title tags. doing this with + # generic code is more complicated than it's worth. the checks above have verified + # that both titles actually exist. + if self._toplevel_tag == 'book': + assert tokens[1].children + assert tokens[4].children + if (maybe_id := cast(str, tokens[0].attrs.get('id', ""))): + maybe_id = "xml:id=" + quoteattr(maybe_id) + return (f'' + f' {self.renderInline(tokens[1].children, options, env)}' + f' {self.renderInline(tokens[4].children, options, env)}' + f' {super().render(tokens[6:], options, env)}' + f'') + + return super().render(tokens, options, env) def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, env: MutableMapping[str, Any]) -> tuple[str, dict[str, str]]: (tag, attrs) = super()._heading_tag(token, tokens, i, options, env) - if self._title_seen: - if token.tag == 'h1': - assert token.map is not None - raise RuntimeError( - "only one title heading (# [text...]) allowed per manual chapter " - f"but found a second in lines [{token.map[0]}..{token.map[1]}]. " - "please remove all such headings except the first, split your " - "chapters, or demote the subsequent headings to (##) or lower.", - token) + # render() has already verified that we don't have supernumerary headings and since the + # book tag is handled specially we can leave the check this simple + if token.tag != 'h1': return (tag, attrs) - self._title_seen = True - return ("chapter", attrs | { + return (self._toplevel_tag, attrs | { 'xmlns': "http://docbook.org/ns/docbook", 'xmlns:xlink': "http://www.w3.org/1999/xlink", }) + def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: int, + options: OptionsDict, env: MutableMapping[str, Any]) -> str: + result = [] + # close existing partintro. the generic render doesn't really need this because + # it doesn't have a concept of structure in the way the manual does. + if self._headings and self._headings[-1] == Heading('part', 1): + result.append("") + self._headings[-1] = self._headings[-1]._replace(partintro_closed=True) + # must nest properly for structural includes. this requires saving at least + # the headings stack, but creating new renderers is cheap and much easier. + r = ManualDocBookRenderer(tag, self._manpage_urls, None) + for (included, path) in token.meta['included']: + try: + result.append(r.render(included, options, env)) + except Exception as e: + raise RuntimeError(f"rendering {path}") from e + return "".join(result) + def included_options(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return cast(str, token.meta['rendered-options']) + # TODO minimize docbook diffs with existing conversions. remove soon. def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, env: MutableMapping[str, Any]) -> str: @@ -76,127 +115,113 @@ class ManualDocBookRenderer(DocBookRenderer): return f"\n{escape(token.content)}" def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, env: MutableMapping[str, Any]) -> str: - # HACK for temporarily being able to replace md-to-db.sh. pandoc used this syntax to - # allow md files to inject arbitrary docbook, and manual chapters use it. - if token.info == '{=docbook}': - return token.content info = f" language={quoteattr(token.info)}" if token.info != "" else "" return f"\n{escape(token.content)}" -class DocBookSectionConverter(BaseConverter): - __renderer__ = ManualDocBookRenderer +class DocBookConverter(Converter): + def __renderer__(self, manpage_urls: Mapping[str, str], + parser: Optional[markdown_it.MarkdownIt]) -> ManualDocBookRenderer: + return ManualDocBookRenderer('book', manpage_urls, parser) - def finalize(self) -> str: - result = [] + _base_paths: list[Path] + _revision: str - for section in self._sections: - id = "id=" + quoteattr(section.id) if section.id is not None else "" - result.append(f'
') - result += section.chapters - result.append(f'
') + def __init__(self, manpage_urls: Mapping[str, str], revision: str): + super().__init__(manpage_urls) + self._revision = revision - return "\n".join(result) - -class ManualFragmentDocBookRenderer(ManualDocBookRenderer): - _tag: str = "chapter" - - def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> tuple[str, dict[str, str]]: - (tag, attrs) = super()._heading_tag(token, tokens, i, options, env) - if token.tag == 'h1': - return (self._tag, attrs | { 'xmlns:xi': "http://www.w3.org/2001/XInclude" }) - return (tag, attrs) - -class DocBookFragmentConverter(Converter): - __renderer__ = ManualFragmentDocBookRenderer - - def convert(self, file: Path, tag: str) -> str: - assert isinstance(self._md.renderer, ManualFragmentDocBookRenderer) + def convert(self, file: Path) -> str: + self._base_paths = [ file ] try: with open(file, 'r') as f: - self._md.renderer._title_seen = False - self._md.renderer._tag = tag return self._render(f.read()) except Exception as e: - raise RuntimeError(f"failed to render manual {tag} {file}") from e + raise RuntimeError(f"failed to render manual {file}") from e + + def _parse(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> list[Token]: + tokens = super()._parse(src, env) + for token in tokens: + if token.type != "fence" or not token.info.startswith("{=include=} "): + continue + typ = token.info[12:].strip() + if typ == 'options': + token.type = 'included_options' + self._parse_options(token) + elif typ in [ 'sections', 'chapters', 'preface', 'parts', 'appendix' ]: + token.type = 'included_' + typ + self._parse_included_blocks(token, env) + else: + raise RuntimeError(f"unsupported structural include type '{typ}'") + return tokens + + def _parse_included_blocks(self, token: Token, env: Optional[MutableMapping[str, Any]]) -> None: + assert token.map + included = token.meta['included'] = [] + for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): + line = line.strip() + path = self._base_paths[-1].parent / line + if path in self._base_paths: + raise RuntimeError(f"circular include found in line {lnum}") + try: + self._base_paths.append(path) + with open(path, 'r') as f: + tokens = self._parse(f.read(), env) + included.append((tokens, path)) + self._base_paths.pop() + except Exception as e: + raise RuntimeError(f"processing included file {path} from line {lnum}") from e + + def _parse_options(self, token: Token) -> None: + assert token.map + + items = {} + for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): + if len(args := line.split(":", 1)) != 2: + raise RuntimeError(f"options directive with no argument in line {lnum}") + (k, v) = (args[0].strip(), args[1].strip()) + if k in items: + raise RuntimeError(f"duplicate options directive {k} in line {lnum}") + items[k] = v + try: + id_prefix = items.pop('id-prefix') + varlist_id = items.pop('list-id') + source = items.pop('source') + except KeyError as e: + raise RuntimeError(f"options directive {e} missing in block at line {token.map[0] + 1}") + if items.keys(): + raise RuntimeError( + f"unsupported options directives in block at line {token.map[0] + 1}", + " ".join(items.keys())) + + try: + conv = options.DocBookConverter( + self._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix) + with open(self._base_paths[-1].parent / source, 'r') as f: + conv.add_options(json.load(f)) + token.meta['rendered-options'] = conv.finalize(fragment=True) + except Exception as e: + raise RuntimeError(f"processing options block in line {token.map[0] + 1}") from e -class Section: - id: Optional[str] = None - chapters: list[str] - - def __init__(self) -> None: - self.chapters = [] - -class SectionAction(argparse.Action): - def __call__(self, parser: argparse.ArgumentParser, ns: argparse.Namespace, - values: Union[str, Sequence[Any], None], opt_str: Optional[str] = None) -> None: - sections = getattr(ns, self.dest) - if sections is None: sections = [] - sections.append(Section()) - setattr(ns, self.dest, sections) - -class SectionIDAction(argparse.Action): - def __call__(self, parser: argparse.ArgumentParser, ns: argparse.Namespace, - values: Union[str, Sequence[Any], None], opt_str: Optional[str] = None) -> None: - sections = getattr(ns, self.dest) - if sections is None: raise argparse.ArgumentError(self, "no active section") - sections[-1].id = cast(str, values) - -class ChaptersAction(argparse.Action): - def __call__(self, parser: argparse.ArgumentParser, ns: argparse.Namespace, - values: Union[str, Sequence[Any], None], opt_str: Optional[str] = None) -> None: - sections = getattr(ns, self.dest) - if sections is None: raise argparse.ArgumentError(self, "no active section") - sections[-1].chapters.extend(map(Path, cast(Sequence[str], values))) - -class SingleFileAction(argparse.Action): - def __call__(self, parser: argparse.ArgumentParser, ns: argparse.Namespace, - values: Union[str, Sequence[Any], None], opt_str: Optional[str] = None) -> None: - assert isinstance(values, Sequence) - chapters = getattr(ns, self.dest) or [] - chapters.append((Path(values[0]), Path(values[1]))) - setattr(ns, self.dest, chapters) - -def _build_cli_db_section(p: argparse.ArgumentParser) -> None: +def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) - p.add_argument("outfile") - p.add_argument("--section", dest="contents", action=SectionAction, nargs=0) - p.add_argument("--section-id", dest="contents", action=SectionIDAction) - p.add_argument("--chapters", dest="contents", action=ChaptersAction, nargs='+') + p.add_argument('--revision', required=True) + p.add_argument('infile', type=Path) + p.add_argument('outfile', type=Path) -def _build_cli_db_fragment(p: argparse.ArgumentParser) -> None: - p.add_argument('--manpage-urls', required=True) - p.add_argument("--chapter", action=SingleFileAction, required=True, nargs=2) - p.add_argument("--section", action=SingleFileAction, required=True, nargs=2) - -def _run_cli_db_section(args: argparse.Namespace) -> None: +def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: - md = DocBookSectionConverter(json.load(manpage_urls)) - for section in args.contents: - md.add_section(section.id, section.chapters) - with open(args.outfile, 'w') as f: - f.write(md.finalize()) - -def _run_cli_db_fragment(args: argparse.Namespace) -> None: - with open(args.manpage_urls, 'r') as manpage_urls: - md = DocBookFragmentConverter(json.load(manpage_urls)) - for kind in [ 'chapter', 'section' ]: - for (target, file) in getattr(args, kind): - converted = md.convert(file, kind) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(converted) + md = DocBookConverter(json.load(manpage_urls), args.revision) + converted = md.convert(args.infile) + args.outfile.write_text(converted) def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) - _build_cli_db_section(formats.add_parser('docbook-section')) - _build_cli_db_fragment(formats.add_parser('docbook-fragment')) + _build_cli_db(formats.add_parser('docbook')) def run_cli(args: argparse.Namespace) -> None: - if args.format == 'docbook-section': - _run_cli_db_section(args) - elif args.format == 'docbook-fragment': - _run_cli_db_fragment(args) + if args.format == 'docbook': + _run_cli_db(args) else: raise RuntimeError('format not hooked up', args) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index b0db410c4818..3cba36140bb4 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -231,10 +231,11 @@ class DocBookConverter(BaseConverter): def _decl_def_footer(self) -> list[str]: return [ "" ] - def finalize(self) -> str: + def finalize(self, *, fragment: bool = False) -> str: result = [] - result.append('') + if not fragment: + result.append('') if self._document_type == 'appendix': result += [ '