From 0997ae1903b0f18cd9e43e4a65ffdd8f1e1ff285 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 19:29:32 +0200 Subject: [PATCH 01/14] nixos/manual: disallow docbook option docs it's been long in the making, and with 23.05 out we can finally disable docbook option docs and default to markdown instead. this brings a massive speed boost in manual and manpage builds, so much so that we may consider enabling user module documentation by default. we don't remove the docbook support code entirely yet because it's a lot all over, and probably better removed in multiple separate changes. --- .github/workflows/compare-manuals.sh | 21 ------ .github/workflows/manual-nixos.yml | 10 +-- .github/workflows/manual-rendering.yml | 64 ------------------- nixos/doc/manual/default.nix | 3 +- .../manual/release-notes/rl-2311.section.md | 2 + nixos/lib/make-options-doc/default.nix | 6 +- nixos/modules/misc/documentation.nix | 22 ++----- 7 files changed, 14 insertions(+), 114 deletions(-) delete mode 100755 .github/workflows/compare-manuals.sh delete mode 100644 .github/workflows/manual-rendering.yml diff --git a/.github/workflows/compare-manuals.sh b/.github/workflows/compare-manuals.sh deleted file mode 100755 index b2cc68c7831d..000000000000 --- a/.github/workflows/compare-manuals.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p html-tidy - -set -euo pipefail -shopt -s inherit_errexit - -normalize() { - tidy \ - --anchor-as-name no \ - --coerce-endtags no \ - --escape-scripts no \ - --fix-backslash no \ - --fix-style-tags no \ - --fix-uri no \ - --indent yes \ - --wrap 0 \ - < "$1" \ - 2> /dev/null -} - -diff -U3 <(normalize "$1") <(normalize "$2") diff --git a/.github/workflows/manual-nixos.yml b/.github/workflows/manual-nixos.yml index 30cecf607d17..2ada5832a7de 100644 --- a/.github/workflows/manual-nixos.yml +++ b/.github/workflows/manual-nixos.yml @@ -27,13 +27,5 @@ jobs: # This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere. name: nixpkgs-ci signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - - name: Building NixOS manual with DocBook options + - name: Building NixOS manual run: NIX_PATH=nixpkgs=$(pwd) nix-build --option restrict-eval true nixos/release.nix -A manual.x86_64-linux - - name: Building NixOS manual with Markdown options - run: | - export NIX_PATH=nixpkgs=$(pwd) - nix-build \ - --option restrict-eval true \ - --arg configuration '{ documentation.nixos.options.allowDocBook = false; }' \ - nixos/release.nix \ - -A manual.x86_64-linux diff --git a/.github/workflows/manual-rendering.yml b/.github/workflows/manual-rendering.yml deleted file mode 100644 index dbaea583ef7d..000000000000 --- a/.github/workflows/manual-rendering.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Check NixOS Manual DocBook rendering against MD rendering" - - -on: - schedule: - # * is a special character in YAML so you have to quote this string - # Check every 24 hours - - cron: '0 0 * * *' - -permissions: - contents: read - -jobs: - check-rendering-equivalence: - permissions: - pull-requests: write # for peter-evans/create-or-update-comment to create or update comment - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v21 - with: - # explicitly enable sandbox - extra_nix_config: sandbox = true - - uses: cachix/cachix-action@v12 - with: - # This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere. - name: nixpkgs-ci - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - - - name: Build DocBook and MD manuals - run: | - export NIX_PATH=nixpkgs=$(pwd) - nix-build \ - --option restrict-eval true \ - -o docbook nixos/release.nix \ - -A manual.x86_64-linux - nix-build \ - --option restrict-eval true \ - --arg configuration '{ documentation.nixos.options.allowDocBook = false; }' \ - -o md nixos/release.nix \ - -A manual.x86_64-linux - - - name: Compare DocBook and MD manuals - id: check - run: | - export NIX_PATH=nixpkgs=$(pwd) - .github/workflows/compare-manuals.sh \ - docbook/share/doc/nixos/options.html \ - md/share/doc/nixos/options.html - - # if the manual can't be built we don't want to notify anyone. - # while this may temporarily hide rendering failures it will be a lot - # less noisy until all nixpkgs pull requests have stopped using - # docbook for option docs. - - name: Comment on failure - uses: peter-evans/create-or-update-comment@v3 - if: ${{ failure() && steps.check.conclusion == 'failure' }} - with: - issue-number: 189318 - body: | - Markdown and DocBook manuals do not agree. - - Check https://github.com/NixOS/nixpkgs/actions/runs/${{ github.run_id }} for details. diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 3052b353ee77..782275b382cd 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -6,10 +6,11 @@ , extraSources ? [] , baseOptionsJSON ? null , warningsAreErrors ? true -, allowDocBook ? true +, allowDocBook ? false , prefix ? ../../.. }: +assert ! allowDocBook; with pkgs; let diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index a86961de6719..700987db224e 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -44,4 +44,6 @@ - A new option was added to the virtualisation module that enables specifying explicitly named network interfaces in QEMU VMs. The existing `virtualisation.vlans` is still supported for cases where the name of the network interface is irrelevant. +- DocBook option documentation is no longer supported, all module documentation now uses markdown. + - `services.nginx` gained a `defaultListen` option at server-level with support for PROXY protocol listeners, also `proxyProtocol` is now exposed in `services.nginx.virtualHosts..listen` option. It is now possible to run PROXY listeners and non-PROXY listeners at a server-level, see [#213510](https://github.com/NixOS/nixpkgs/pull/213510/) for more details. diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index a2385582a014..e4d9a6ebcc71 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -39,12 +39,16 @@ # allow docbook option docs if `true`. only markdown documentation is allowed when set to # `false`, and a different renderer may be used with different bugs and performance # characteristics but (hopefully) indistinguishable output. -, allowDocBook ? true +# deprecated since 23.11. +# TODO remove in a while. +, allowDocBook ? false # whether lib.mdDoc is required for descriptions to be read as markdown. # !!! when this is eventually flipped to true, `lib.doRename` should also default to emitting Markdown , markdownByDefault ? false }: +assert ! allowDocBook; + let rawOpts = lib.optionAttrSetToDocList options; transformedOpts = map transformOptions rawOpts; diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix index 31486a2216ad..e6a56a8cdf28 100644 --- a/nixos/modules/misc/documentation.nix +++ b/nixos/modules/misc/documentation.nix @@ -107,7 +107,7 @@ let } >&2 ''; - inherit (cfg.nixos.options) warningsAreErrors allowDocBook; + inherit (cfg.nixos.options) warningsAreErrors; }; @@ -160,6 +160,9 @@ in (mkRenamedOptionModule [ "programs" "info" "enable" ] [ "documentation" "info" "enable" ]) (mkRenamedOptionModule [ "programs" "man" "enable" ] [ "documentation" "man" "enable" ]) (mkRenamedOptionModule [ "services" "nixosManual" "enable" ] [ "documentation" "nixos" "enable" ]) + (mkRemovedOptionModule + [ "documentation" "nixos" "options" "allowDocBook" ] + "DocBook option documentation is no longer supported") ]; options = { @@ -264,23 +267,6 @@ in ''; }; - nixos.options.allowDocBook = mkOption { - type = types.bool; - default = true; - description = lib.mdDoc '' - Whether to allow DocBook option docs. When set to `false` all option using - DocBook documentation will cause a manual build error; additionally a new - renderer may be used. - - ::: {.note} - The `false` setting for this option is not yet fully supported. While it - should work fine and produce the same output as the previous toolchain - using DocBook it may not work in all circumstances. Whether markdown option - documentation is allowed is independent of this option. - ::: - ''; - }; - nixos.options.warningsAreErrors = mkOption { type = types.bool; default = true; From 7542a1aa8fde98d8ff2792f852af19a8bc798842 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 19:51:46 +0200 Subject: [PATCH 02/14] lib/options: remove literalDocBook no longer supported. warning when used would not be appropriate, and docbook has been on the way out for long enough that throwing an error should not be necessary either. --- lib/default.nix | 2 +- lib/options.nix | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index 8fea4b8ad637..73b8ad871544 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -138,7 +138,7 @@ let mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption getValues getFiles optionAttrSetToDocList optionAttrSetToDocList' - scrubOptionValue literalExpression literalExample literalDocBook + scrubOptionValue literalExpression literalExample showOption showOptionWithDefLocs showFiles unknownModule mkOption mkPackageOption mkPackageOptionMD mdDoc literalMD; diff --git a/lib/options.nix b/lib/options.nix index af7914bb5137..11a3984aa47e 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -344,19 +344,7 @@ rec { if ! isString text then throw "literalExpression expects a string." else { _type = "literalExpression"; inherit text; }; - literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalDocBook for a non-Nix description." literalExpression; - - - /* For use in the `defaultText` and `example` option attributes. Causes the - given DocBook text to be inserted verbatim in the documentation, for when - a `literalExpression` would be too hard to read. - */ - literalDocBook = text: - if ! isString text then throw "literalDocBook expects a string." - else - lib.warnIf (lib.isInOldestRelease 2211) - "literalDocBook is deprecated, use literalMD instead" - { _type = "literalDocBook"; inherit text; }; + literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalMD for a non-Nix description." literalExpression; /* Transition marker for documentation that's already migrated to markdown syntax. From 5f35ae16ec0ff4bb71776e3538fde2f561310b64 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 20:19:33 +0200 Subject: [PATCH 03/14] nixos-render-docs: remove literalDocBook support with literalDocBook itself gone we can also remove the support code in nixos-render-docs. --- .../src/nixos_render_docs/options.py | 41 +++---------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 06e5f9711216..6f7c4f7400a4 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -216,12 +216,6 @@ class DocBookConverter(BaseConverter[OptionsDocBookRenderer]): def _parallel_render_init_worker(cls, a: Any) -> DocBookConverter: return cls(*a) - def _render_code(self, option: dict[str, Any], key: str) -> list[str]: - if lit := option_is(option, key, 'literalDocBook'): - return [ f"{key.capitalize()}: {lit['text']}" ] - else: - return super()._render_code(option, key) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: if isinstance(desc, str) and not self._markdown_by_default: return [ f"{desc}" ] @@ -327,14 +321,11 @@ class ManpageConverter(BaseConverter[OptionsManpageRenderer]): return super().add_options(options) def _render_code(self, option: dict[str, Any], key: str) -> list[str]: - if lit := option_is(option, key, 'literalDocBook'): - raise RuntimeError("can't render manpages in the presence of docbook") - else: - try: - self._renderer.inline_code_is_quoted = False - return super()._render_code(option, key) - finally: - self._renderer.inline_code_is_quoted = True + try: + self._renderer.inline_code_is_quoted = False + return super()._render_code(option, key) + finally: + self._renderer.inline_code_is_quoted = True def _render_description(self, desc: str | dict[str, Any]) -> list[str]: if isinstance(desc, str) and not self._markdown_by_default: @@ -431,14 +422,6 @@ class CommonMarkConverter(BaseConverter[OptionsCommonMarkRenderer]): def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter: return cls(*a) - def _render_code(self, option: dict[str, Any], key: str) -> list[str]: - # NOTE this duplicates the old direct-paste behavior, even if it is somewhat - # incorrect, since users rely on it. - if lit := option_is(option, key, 'literalDocBook'): - return [ f"*{key.capitalize()}:* {lit['text']}" ] - else: - return super()._render_code(option, key) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: # NOTE this duplicates the old direct-paste behavior, even if it is somewhat # incorrect, since users rely on it. @@ -487,14 +470,6 @@ class AsciiDocConverter(BaseConverter[OptionsAsciiDocRenderer]): def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter: return cls(*a) - def _render_code(self, option: dict[str, Any], key: str) -> list[str]: - # NOTE this duplicates the old direct-paste behavior, even if it is somewhat - # incorrect, since users rely on it. - if lit := option_is(option, key, 'literalDocBook'): - return [ f"*{key.capitalize()}:* {lit['text']}" ] - else: - return super()._render_code(option, key) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: # NOTE this duplicates the old direct-paste behavior, even if it is somewhat # incorrect, since users rely on it. @@ -557,12 +532,6 @@ class HTMLConverter(BaseConverter[OptionsHTMLRenderer]): def _parallel_render_init_worker(cls, a: Any) -> HTMLConverter: return cls(*a) - def _render_code(self, option: dict[str, Any], key: str) -> list[str]: - if lit := option_is(option, key, 'literalDocBook'): - raise RuntimeError("can't render html in the presence of docbook") - else: - return super()._render_code(option, key) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: if isinstance(desc, str) and not self._markdown_by_default: raise RuntimeError("can't render html in the presence of docbook") From c01244394c0b7278c507d6d5de9e4a78db85d1fa Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 20:07:13 +0200 Subject: [PATCH 04/14] nixos/make-options-doc: force markdownByDefault with docbook no longer supported we can default to markdown option docs. we'll keep the parameter around for a bit to not break external users who set it to true. we don't know of any users that do, so the deprecation period may be rather short for this one. --- nixos/lib/make-options-doc/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index e4d9a6ebcc71..669bc221ae27 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -43,11 +43,12 @@ # TODO remove in a while. , allowDocBook ? false # whether lib.mdDoc is required for descriptions to be read as markdown. -# !!! when this is eventually flipped to true, `lib.doRename` should also default to emitting Markdown -, markdownByDefault ? false +# deprecated since 23.11. +# TODO remove in a while. +, markdownByDefault ? true }: -assert ! allowDocBook; +assert markdownByDefault && ! allowDocBook; let rawOpts = lib.optionAttrSetToDocList options; From 34eeac55449400f866573a3c9027c1d336db40a4 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 20:09:54 +0200 Subject: [PATCH 05/14] nixos-render-docs: default to markdown for options docbook is now gone and we can flip the defaults. we won't keep the command line args around (unlike the make-options-docs argument) because nixos-render-docs should not be considered an exposed API. --- nixos/lib/make-options-doc/default.nix | 1 - .../src/nixos_render_docs/manual.py | 4 +- .../src/nixos_render_docs/options.py | 90 +++++-------------- .../src/tests/test_options.py | 2 +- 4 files changed, 23 insertions(+), 74 deletions(-) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 669bc221ae27..db0e0a451937 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -169,7 +169,6 @@ in rec { --document-type ${lib.escapeShellArg documentType} \ --varlist-id ${lib.escapeShellArg variablelistId} \ --id-prefix ${lib.escapeShellArg optionIdPrefix} \ - ${lib.optionalString markdownByDefault "--markdown-by-default"} \ ${optionsJSON}/share/doc/nixos/options.json \ options.xml diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 1963989d5365..f1ba1535ad29 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -209,7 +209,7 @@ class ManualDocBookRenderer(RendererMixin, DocBookRenderer): raise RuntimeError(f"rendering {path}") from e return "".join(result) def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: - conv = options.DocBookConverter(self._manpage_urls, self._revision, False, 'fragment', + conv = options.DocBookConverter(self._manpage_urls, self._revision, 'fragment', token.meta['list-id'], token.meta['id-prefix']) conv.add_options(token.meta['source']) return conv.finalize(fragment=True) @@ -471,7 +471,7 @@ class ManualHTMLRenderer(RendererMixin, HTMLRenderer): return "".join(outer) def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: - conv = options.HTMLConverter(self._manpage_urls, self._revision, False, + conv = options.HTMLConverter(self._manpage_urls, self._revision, token.meta['list-id'], token.meta['id-prefix'], self._xref_targets) conv.add_options(token.meta['source']) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 6f7c4f7400a4..469452bbffb2 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -38,11 +38,10 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]): _options: dict[str, RenderedOption] - def __init__(self, revision: str, markdown_by_default: bool): + def __init__(self, revision: str): super().__init__() self._options = {} self._revision = revision - self._markdown_by_default = markdown_by_default def _sorted_options(self) -> list[tuple[str, RenderedOption]]: keys = list(self._options.keys()) @@ -107,7 +106,7 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]): return [] def _render_description(self, desc: str | dict[str, str]) -> list[str]: - if isinstance(desc, str) and self._markdown_by_default: + if isinstance(desc, str): return [ self._render(desc) ] if desc else [] elif isinstance(desc, dict) and desc.get('_type') == 'mdDoc': return [ self._render(desc['text']) ] if desc['text'] else [] @@ -199,29 +198,22 @@ class DocBookConverter(BaseConverter[OptionsDocBookRenderer]): def __init__(self, manpage_urls: Mapping[str, str], revision: str, - markdown_by_default: bool, document_type: str, varlist_id: str, id_prefix: str): - super().__init__(revision, markdown_by_default) + super().__init__(revision) self._renderer = OptionsDocBookRenderer(manpage_urls) self._document_type = document_type self._varlist_id = varlist_id self._id_prefix = id_prefix def _parallel_render_prepare(self) -> Any: - return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, self._document_type, + return (self._renderer._manpage_urls, self._revision, self._document_type, self._varlist_id, self._id_prefix) @classmethod def _parallel_render_init_worker(cls, a: Any) -> DocBookConverter: return cls(*a) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: - if isinstance(desc, str) and not self._markdown_by_default: - return [ f"{desc}" ] - else: - return super()._render_description(desc) - def _related_packages_header(self) -> list[str]: return [ "", @@ -295,19 +287,19 @@ class ManpageConverter(BaseConverter[OptionsManpageRenderer]): _options_by_id: dict[str, str] _links_in_last_description: Optional[list[str]] = None - def __init__(self, revision: str, markdown_by_default: bool, + def __init__(self, revision: str, *, # only for parallel rendering _options_by_id: Optional[dict[str, str]] = None): - super().__init__(revision, markdown_by_default) + super().__init__(revision) self._options_by_id = _options_by_id or {} self._renderer = OptionsManpageRenderer({}, self._options_by_id) def _parallel_render_prepare(self) -> Any: - return ((self._revision, self._markdown_by_default), { '_options_by_id': self._options_by_id }) + return (self._revision, { '_options_by_id': self._options_by_id }) @classmethod def _parallel_render_init_worker(cls, a: Any) -> ManpageConverter: - return cls(*a[0], **a[1]) + return cls(a[0], **a[1]) def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption: links = self._renderer.link_footnotes = [] @@ -327,12 +319,6 @@ class ManpageConverter(BaseConverter[OptionsManpageRenderer]): finally: self._renderer.inline_code_is_quoted = True - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: - if isinstance(desc, str) and not self._markdown_by_default: - raise RuntimeError("can't render manpages in the presence of docbook") - else: - return super()._render_description(desc) - def _related_packages_header(self) -> list[str]: return [ '\\fIRelated packages:\\fP', @@ -412,24 +398,16 @@ class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer): class CommonMarkConverter(BaseConverter[OptionsCommonMarkRenderer]): __option_block_separator__ = "" - def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool): - super().__init__(revision, markdown_by_default) + def __init__(self, manpage_urls: Mapping[str, str], revision: str): + super().__init__(revision) self._renderer = OptionsCommonMarkRenderer(manpage_urls) def _parallel_render_prepare(self) -> Any: - return (self._renderer._manpage_urls, self._revision, self._markdown_by_default) + return (self._renderer._manpage_urls, self._revision) @classmethod def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter: return cls(*a) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: - # NOTE this duplicates the old direct-paste behavior, even if it is somewhat - # incorrect, since users rely on it. - if isinstance(desc, str) and not self._markdown_by_default: - return [ desc ] - else: - return super()._render_description(desc) - def _related_packages_header(self) -> list[str]: return [ "*Related packages:*" ] @@ -460,24 +438,16 @@ class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer): class AsciiDocConverter(BaseConverter[OptionsAsciiDocRenderer]): __option_block_separator__ = "" - def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool): - super().__init__(revision, markdown_by_default) + def __init__(self, manpage_urls: Mapping[str, str], revision: str): + super().__init__(revision) self._renderer = OptionsAsciiDocRenderer(manpage_urls) def _parallel_render_prepare(self) -> Any: - return (self._renderer._manpage_urls, self._revision, self._markdown_by_default) + return (self._renderer._manpage_urls, self._revision) @classmethod def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter: return cls(*a) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: - # NOTE this duplicates the old direct-paste behavior, even if it is somewhat - # incorrect, since users rely on it. - if isinstance(desc, str) and not self._markdown_by_default: - return [ desc ] - else: - return super()._render_description(desc) - def _related_packages_header(self) -> list[str]: return [ "__Related packages:__" ] @@ -517,27 +487,21 @@ class OptionsHTMLRenderer(OptionDocsRestrictions, HTMLRenderer): class HTMLConverter(BaseConverter[OptionsHTMLRenderer]): __option_block_separator__ = "" - def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool, + def __init__(self, manpage_urls: Mapping[str, str], revision: str, varlist_id: str, id_prefix: str, xref_targets: Mapping[str, XrefTarget]): - super().__init__(revision, markdown_by_default) + super().__init__(revision) self._xref_targets = xref_targets self._varlist_id = varlist_id self._id_prefix = id_prefix self._renderer = OptionsHTMLRenderer(manpage_urls, self._xref_targets) def _parallel_render_prepare(self) -> Any: - return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, + return (self._renderer._manpage_urls, self._revision, self._varlist_id, self._id_prefix, self._xref_targets) @classmethod def _parallel_render_init_worker(cls, a: Any) -> HTMLConverter: return cls(*a) - def _render_description(self, desc: str | dict[str, Any]) -> list[str]: - if isinstance(desc, str) and not self._markdown_by_default: - raise RuntimeError("can't render html in the presence of docbook") - else: - return super()._render_description(desc) - def _related_packages_header(self) -> list[str]: return [ '

Related packages:

', @@ -605,7 +569,6 @@ def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--document-type', required=True) p.add_argument('--varlist-id', required=True) p.add_argument('--id-prefix', required=True) - p.add_argument('--markdown-by-default', default=False, action='store_true') p.add_argument("infile") p.add_argument("outfile") @@ -617,14 +580,12 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None: def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) - p.add_argument('--markdown-by-default', default=False, action='store_true') p.add_argument("infile") p.add_argument("outfile") def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) - p.add_argument('--markdown-by-default', default=False, action='store_true') p.add_argument("infile") p.add_argument("outfile") @@ -633,7 +594,6 @@ def _run_cli_db(args: argparse.Namespace) -> None: md = DocBookConverter( json.load(manpage_urls), revision = args.revision, - markdown_by_default = args.markdown_by_default, document_type = args.document_type, varlist_id = args.varlist_id, id_prefix = args.id_prefix) @@ -644,11 +604,7 @@ def _run_cli_db(args: argparse.Namespace) -> None: f.write(md.finalize()) def _run_cli_manpage(args: argparse.Namespace) -> None: - md = ManpageConverter( - revision = args.revision, - # manpage rendering only works if there's no docbook, so we can - # also set markdown_by_default with no ill effects. - markdown_by_default = True) + md = ManpageConverter(revision = args.revision) with open(args.infile, 'r') as f: md.add_options(json.load(f)) @@ -657,10 +613,7 @@ def _run_cli_manpage(args: argparse.Namespace) -> None: def _run_cli_commonmark(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: - md = CommonMarkConverter( - json.load(manpage_urls), - revision = args.revision, - markdown_by_default = args.markdown_by_default) + md = CommonMarkConverter(json.load(manpage_urls), revision = args.revision) with open(args.infile, 'r') as f: md.add_options(json.load(f)) @@ -669,10 +622,7 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None: def _run_cli_asciidoc(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: - md = AsciiDocConverter( - json.load(manpage_urls), - revision = args.revision, - markdown_by_default = args.markdown_by_default) + md = AsciiDocConverter(json.load(manpage_urls), revision = args.revision) with open(args.infile, 'r') as f: md.add_options(json.load(f)) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_options.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_options.py index 9608ed639218..fed0b1a17ac1 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_options.py @@ -4,7 +4,7 @@ from markdown_it.token import Token import pytest def test_option_headings() -> None: - c = nixos_render_docs.options.DocBookConverter({}, 'local', False, 'none', 'vars', 'opt-') + c = nixos_render_docs.options.DocBookConverter({}, 'local', 'none', 'vars', 'opt-') with pytest.raises(RuntimeError) as exc: c._render("# foo") assert exc.value.args[0] == 'md token not supported in options doc' From af1f07ff03c59b081ff29f069f6c28009dfffad7 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 13 Jun 2023 14:05:40 +0200 Subject: [PATCH 06/14] nixos/make-options-doc: check for manual paths in options.json since we no longer use the docbook path the check there will no longer fire. add one to optionsJSON to not lose this functionality. --- nixos/lib/make-options-doc/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index db0e0a451937..5cfc24533c23 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -143,6 +143,14 @@ in rec { $baseJSON $options \ > $dst/options.json + if grep /nixpkgs/nixos/modules $dst/options.json; then + echo "The manual appears to depend on the location of Nixpkgs, which is bad" + echo "since this prevents sharing via the NixOS channel. This is typically" + echo "caused by an option default that refers to a relative path (see above" + echo "for hints about the offending path)." + exit 1 + fi + brotli -9 < $dst/options.json > $dst/options.json.br mkdir -p $out/nix-support From 1418c986b00d2386baac359ee80e004fdba5083c Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 13 Jun 2023 14:06:38 +0200 Subject: [PATCH 07/14] nixos/make-options-doc: remove options postprocessing with everything being rendered from markdown now we no longer need to postprocess any options.xml that may be requested from elsewhere. we'll don't need to keep the module path check either since that's done by optionsJSON now. --- nixos/lib/make-options-doc/default.nix | 13 +- .../postprocess-option-descriptions.xsl | 115 ------------------ 2 files changed, 1 insertion(+), 127 deletions(-) delete mode 100644 nixos/lib/make-options-doc/postprocess-option-descriptions.xsl diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 5cfc24533c23..4ba8a9d95b36 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -178,17 +178,6 @@ in rec { --varlist-id ${lib.escapeShellArg variablelistId} \ --id-prefix ${lib.escapeShellArg optionIdPrefix} \ ${optionsJSON}/share/doc/nixos/options.json \ - options.xml - - if grep /nixpkgs/nixos/modules options.xml; then - echo "The manual appears to depend on the location of Nixpkgs, which is bad" - echo "since this prevents sharing via the NixOS channel. This is typically" - echo "caused by an option default that refers to a relative path (see above" - echo "for hints about the offending path)." - exit 1 - fi - - ${pkgs.libxslt.bin}/bin/xsltproc \ - -o "$out" ${./postprocess-option-descriptions.xsl} options.xml + "$out" ''; } diff --git a/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl b/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl deleted file mode 100644 index 1201c7612c2e..000000000000 --- a/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl +++ /dev/null @@ -1,115 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 20152b4269356ab04cab7b3c7b0df3fea3455637 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 20:36:29 +0200 Subject: [PATCH 08/14] nixos/doc: remove docbook options compatibility no longer needed or useful, and may even produce false positives now that markdown is the default language for option docs. --- nixos/doc/manual/default.nix | 84 +++++++------------------ nixos/lib/make-options-doc/default.nix | 1 - nixos/lib/make-options-doc/mergeJSON.py | 60 ------------------ 3 files changed, 23 insertions(+), 122 deletions(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 782275b382cd..cc28fe6d16ca 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -202,14 +202,8 @@ in rec { # Generate the NixOS manual. manualHTML = runCommand "nixos-manual-html" - { nativeBuildInputs = - if allowDocBook then [ - buildPackages.libxml2.bin - buildPackages.libxslt.bin - ] else [ - buildPackages.nixos-render-docs - ]; - inputs = lib.optionals (! allowDocBook) (lib.sourceFilesBySuffices ./. [ ".md" ]); + { nativeBuildInputs = [ buildPackages.nixos-render-docs ]; + inputs = lib.sourceFilesBySuffices ./. [ ".md" ]; meta.description = "The NixOS manual in HTML format"; allowedReferences = ["out"]; } @@ -222,38 +216,23 @@ in rec { cp ${../../../doc/overrides.css} $dst/overrides.css cp -r ${pkgs.documentation-highlighter} $dst/highlightjs - ${if allowDocBook then '' - xsltproc \ - ${manualXsltprocOptions} \ - --stringparam id.warnings "1" \ - --nonet --output $dst/ \ - ${docbook_xsl_ns}/xml/xsl/docbook/xhtml/chunktoc.xsl \ - ${manual-combined}/manual-combined.xml \ - |& tee xsltproc.out - grep "^ID recommended on" xsltproc.out &>/dev/null && echo "error: some IDs are missing" && false - rm xsltproc.out + ${prepareManualFromMD} - mkdir -p $dst/images/callouts - cp ${docbook_xsl_ns}/xml/xsl/docbook/images/callouts/*.svg $dst/images/callouts/ - '' else '' - ${prepareManualFromMD} - - # TODO generator is set like this because the docbook/md manual compare workflow will - # trigger if it's different - nixos-render-docs -j $NIX_BUILD_CORES manual html \ - --manpage-urls ${manpageUrls} \ - --revision ${lib.escapeShellArg revision} \ - --generator "DocBook XSL Stylesheets V${docbook_xsl_ns.version}" \ - --stylesheet style.css \ - --stylesheet overrides.css \ - --stylesheet highlightjs/mono-blue.css \ - --script ./highlightjs/highlight.pack.js \ - --script ./highlightjs/loader.js \ - --toc-depth 1 \ - --chunk-toc-depth 1 \ - ./manual.md \ - $dst/index.html - ''} + # TODO generator is set like this because the docbook/md manual compare workflow will + # trigger if it's different + nixos-render-docs -j $NIX_BUILD_CORES manual html \ + --manpage-urls ${manpageUrls} \ + --revision ${lib.escapeShellArg revision} \ + --generator "DocBook XSL Stylesheets V${docbook_xsl_ns.version}" \ + --stylesheet style.css \ + --stylesheet overrides.css \ + --stylesheet highlightjs/mono-blue.css \ + --script ./highlightjs/highlight.pack.js \ + --script ./highlightjs/loader.js \ + --toc-depth 1 \ + --chunk-toc-depth 1 \ + ./manual.md \ + $dst/index.html mkdir -p $out/nix-support echo "nix-build out $out" >> $out/nix-support/hydra-build-products @@ -319,10 +298,6 @@ in rec { manpages = runCommand "nixos-manpages" { nativeBuildInputs = [ buildPackages.installShellFiles - ] ++ lib.optionals allowDocBook [ - buildPackages.libxml2.bin - buildPackages.libxslt.bin - ] ++ lib.optionals (! allowDocBook) [ buildPackages.nixos-render-docs ]; allowedReferences = ["out"]; @@ -331,24 +306,11 @@ in rec { # Generate manpages. mkdir -p $out/share/man/man8 installManPage ${./manpages}/* - ${if allowDocBook - then '' - xsltproc --nonet \ - --maxdepth 6000 \ - --param man.output.in.separate.dir 1 \ - --param man.output.base.dir "'$out/share/man/'" \ - --param man.endnotes.are.numbered 0 \ - --param man.break.after.slash 1 \ - ${docbook_xsl_ns}/xml/xsl/docbook/manpages/docbook.xsl \ - ${manpages-combined} - '' - else '' - mkdir -p $out/share/man/man5 - nixos-render-docs -j $NIX_BUILD_CORES options manpage \ - --revision ${lib.escapeShellArg revision} \ - ${optionsJSON}/share/doc/nixos/options.json \ - $out/share/man/man5/configuration.nix.5 - ''} + mkdir -p $out/share/man/man5 + nixos-render-docs -j $NIX_BUILD_CORES options manpage \ + --revision ${lib.escapeShellArg revision} \ + ${optionsJSON}/share/doc/nixos/options.json \ + $out/share/man/man5/configuration.nix.5 ''; } diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 4ba8a9d95b36..26819597a1f9 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -139,7 +139,6 @@ in rec { TOUCH_IF_DB=$dst/.used-docbook \ python ${./mergeJSON.py} \ ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \ - ${if allowDocBook then "--warn-on-docbook" else "--error-on-docbook"} \ $baseJSON $options \ > $dst/options.json diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py index b4f72b8a3fdc..4be83fcb827b 100644 --- a/nixos/lib/make-options-doc/mergeJSON.py +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -43,19 +43,11 @@ def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: return result warningsAreErrors = False -warnOnDocbook = False -errorOnDocbook = False optOffset = 0 for arg in sys.argv[1:]: if arg == "--warnings-are-errors": optOffset += 1 warningsAreErrors = True - if arg == "--warn-on-docbook": - optOffset += 1 - warnOnDocbook = True - elif arg == "--error-on-docbook": - optOffset += 1 - errorOnDocbook = True options = pivot(json.load(open(sys.argv[1 + optOffset], 'r'))) overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r'))) @@ -84,38 +76,10 @@ for (k, v) in overrides.items(): severity = "error" if warningsAreErrors else "warning" -def is_docbook(o, key): - val = o.get(key, {}) - if not isinstance(val, dict): - return False - return val.get('_type', '') == 'literalDocBook' - # check that every option has a description hasWarnings = False hasErrors = False -hasDocBook = False for (k, v) in options.items(): - if warnOnDocbook or errorOnDocbook: - kind = "error" if errorOnDocbook else "warning" - if isinstance(v.value.get('description', {}), str): - hasErrors |= errorOnDocbook - hasDocBook = True - print( - f"\x1b[1;31m{kind}: option {v.name} description uses DocBook\x1b[0m", - file=sys.stderr) - elif is_docbook(v.value, 'defaultText'): - hasErrors |= errorOnDocbook - hasDocBook = True - print( - f"\x1b[1;31m{kind}: option {v.name} default uses DocBook\x1b[0m", - file=sys.stderr) - elif is_docbook(v.value, 'example'): - hasErrors |= errorOnDocbook - hasDocBook = True - print( - f"\x1b[1;31m{kind}: option {v.name} example uses DocBook\x1b[0m", - file=sys.stderr) - if v.value.get('description', None) is None: hasWarnings = True print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) @@ -126,30 +90,6 @@ for (k, v) in options.items(): f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " + "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr) -if hasDocBook: - (why, what) = ( - ("disallowed for in-tree modules", "contribution") if errorOnDocbook - else ("deprecated for option documentation", "module") - ) - print("Explanation: The documentation contains descriptions, examples, or defaults written in DocBook. " + - "NixOS is in the process of migrating from DocBook to Markdown, and " + - f"DocBook is {why}. To change your {what} to "+ - "use Markdown, apply mdDoc and literalMD and use the *MD variants of option creation " + - "functions where they are available. For example:\n" + - "\n" + - " example.foo = mkOption {\n" + - " description = lib.mdDoc ''your description'';\n" + - " defaultText = lib.literalMD ''your description of default'';\n" + - " };\n" + - "\n" + - " example.enable = mkEnableOption (lib.mdDoc ''your thing'');\n" + - " example.package = mkPackageOptionMD pkgs \"your-package\" {};\n" + - " imports = [ (mkAliasOptionModuleMD [ \"example\" \"args\" ] [ \"example\" \"settings\" ]) ];", - file = sys.stderr) - with open(os.getenv('TOUCH_IF_DB'), 'x'): - # just make sure it exists - pass - if hasErrors: sys.exit(1) if hasWarnings and warningsAreErrors: From d36f950d405369abca5144b14a201bc25c4a080a Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 20:27:54 +0200 Subject: [PATCH 09/14] lib: turn *MD functions into aliases with docbook gone and MD the default these aren't needed any more. we can't remove them yet because there's thousands of uses, but maybe some day we can. --- lib/modules.nix | 19 +++++++------------ lib/options.nix | 19 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 4dc8c663b2fe..5821aeab20c7 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -1146,14 +1146,11 @@ let use = id; }; - /* Transitional version of mkAliasOptionModule that uses MD docs. */ - mkAliasOptionModuleMD = from: to: doRename { - inherit from to; - visible = true; - warn = false; - use = id; - markdown = true; - }; + /* Transitional version of mkAliasOptionModule that uses MD docs. + + This function is no longer necessary and merely an alias of `mkAliasOptionModule`. + */ + mkAliasOptionModuleMD = mkAliasOptionModule; /* mkDerivedConfig : Option a -> (a -> Definition b) -> Definition b @@ -1175,7 +1172,7 @@ let (opt.highestPrio or defaultOverridePriority) (f opt.value); - doRename = { from, to, visible, warn, use, withPriority ? true, markdown ? false }: + doRename = { from, to, visible, warn, use, withPriority ? true }: { config, options, ... }: let fromOpt = getAttrFromPath from options; @@ -1186,9 +1183,7 @@ let { options = setAttrByPath from (mkOption { inherit visible; - description = if markdown - then lib.mdDoc "Alias of {option}`${showOption to}`." - else "Alias of ."; + description = "Alias of {option}`${showOption to}`."; apply = x: use (toOf config); } // optionalAttrs (toType != null) { type = toType; diff --git a/lib/options.nix b/lib/options.nix index 11a3984aa47e..c42bc1e6c67e 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -100,10 +100,7 @@ rec { name: mkOption { default = false; example = true; - description = - if name ? _type && name._type == "mdDoc" - then lib.mdDoc "Whether to enable ${name.text}." - else "Whether to enable ${name}."; + description = "Whether to enable ${name}."; type = lib.types.bool; }; @@ -185,10 +182,10 @@ rec { (if isList example then "pkgs." + concatStringsSep "." example else example); }); - /* Like mkPackageOption, but emit an mdDoc description instead of DocBook. */ - mkPackageOptionMD = pkgs: name: extra: - let option = mkPackageOption pkgs name extra; - in option // { description = lib.mdDoc option.description; }; + /* Alias of mkPackageOption. Previously used to create options with markdown + documentation, which is no longer required. + */ + mkPackageOptionMD = mkPackageOption; /* This option accepts anything, but it does not produce any result. @@ -347,11 +344,9 @@ rec { literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalMD for a non-Nix description." literalExpression; /* Transition marker for documentation that's already migrated to markdown - syntax. + syntax. This is a no-op and no longer needed. */ - mdDoc = text: - if ! isString text then throw "mdDoc expects a string." - else { _type = "mdDoc"; inherit text; }; + mdDoc = lib.id; /* For use in the `defaultText` and `example` option attributes. Causes the given MD text to be inserted verbatim in the documentation, for when From f52f531a4e4791f79902ed3f4162bfd8ae899c01 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 13 Jun 2023 12:21:38 +0200 Subject: [PATCH 10/14] nixos/make-options-doc: deprecate docbook outputs they're no longer necessary for us and will almost definitely start to rot now (like commonmark and asciidoc outputs did previously). most existing users seem to take the docbook output and run it through pandoc to generate html, those can easily migrate to use commonmark instead. other users will hopefully pipe up when they notice that things they rely on are going away. optionsUsedDocbook has only been around for one release and only exposed to allow other places to generate warnings, so that does not deserve such precautions. --- nixos/doc/manual/default.nix | 2 +- nixos/lib/make-options-doc/default.nix | 37 +++++++++++--------------- nixos/modules/misc/documentation.nix | 8 ------ 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index cc28fe6d16ca..ef7803fb75b0 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -198,7 +198,7 @@ let ''; in rec { - inherit (optionsDoc) optionsJSON optionsNix optionsDocBook optionsUsedDocbook; + inherit (optionsDoc) optionsJSON optionsNix optionsDocBook; # Generate the NixOS manual. manualHTML = runCommand "nixos-manual-html" diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 26819597a1f9..99515b5b8276 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -157,26 +157,19 @@ in rec { echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products ''; - optionsUsedDocbook = pkgs.runCommand "options-used-docbook" {} '' - if [ -e ${optionsJSON}/share/doc/nixos/.used-docbook ]; then - echo 1 - else - echo 0 - fi >"$out" - ''; - - optionsDocBook = pkgs.runCommand "options-docbook.xml" { - nativeBuildInputs = [ - pkgs.nixos-render-docs - ]; - } '' - nixos-render-docs -j $NIX_BUILD_CORES options docbook \ - --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ - --revision ${lib.escapeShellArg revision} \ - --document-type ${lib.escapeShellArg documentType} \ - --varlist-id ${lib.escapeShellArg variablelistId} \ - --id-prefix ${lib.escapeShellArg optionIdPrefix} \ - ${optionsJSON}/share/doc/nixos/options.json \ - "$out" - ''; + optionsDocBook = lib.warn "optionsDocBook is deprecated since 23.11 and will be removed in 24.05" + (pkgs.runCommand "options-docbook.xml" { + nativeBuildInputs = [ + pkgs.nixos-render-docs + ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options docbook \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ + --document-type ${lib.escapeShellArg documentType} \ + --varlist-id ${lib.escapeShellArg variablelistId} \ + --id-prefix ${lib.escapeShellArg optionIdPrefix} \ + ${optionsJSON}/share/doc/nixos/options.json \ + "$out" + ''); } diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix index e6a56a8cdf28..820450e3ce2a 100644 --- a/nixos/modules/misc/documentation.nix +++ b/nixos/modules/misc/documentation.nix @@ -345,14 +345,6 @@ in (mkIf cfg.nixos.enable { system.build.manual = manual; - system.activationScripts.check-manual-docbook = '' - if [[ $(cat ${manual.optionsUsedDocbook}) = 1 ]]; then - echo -e "\e[31;1mwarning\e[0m: This configuration contains option documentation in docbook." \ - "Support for docbook is deprecated and will be removed after NixOS 23.05." \ - "See nix-store --read-log ${builtins.unsafeDiscardStringContext manual.optionsJSON.drvPath}" - fi - ''; - environment.systemPackages = [] ++ optional cfg.man.enable manual.manpages ++ optionals cfg.doc.enable [ manual.manualHTML nixos-help ]; From 426903d2fba217314309e3002e0247d3a772564e Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 13 Jun 2023 12:38:22 +0200 Subject: [PATCH 11/14] nixos/manual: remove docbook intermediates they're no longer used for anything. --- nixos/doc/manual/default.nix | 117 +-------------------------------- nixos/doc/manual/man-pages.xml | 46 ------------- 2 files changed, 1 insertion(+), 162 deletions(-) delete mode 100644 nixos/doc/manual/man-pages.xml diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index ef7803fb75b0..03f0d26fdf93 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -6,11 +6,9 @@ , extraSources ? [] , baseOptionsJSON ? null , warningsAreErrors ? true -, allowDocBook ? false , prefix ? ../../.. }: -assert ! allowDocBook; with pkgs; let @@ -18,10 +16,6 @@ let lib = pkgs.lib; - docbook_xsl_ns = pkgs.docbook-xsl-ns.override { - withManOptDedupPatch = true; - }; - manpageUrls = pkgs.path + "/doc/manpage-urls.json"; # We need to strip references to /nix/store/* from options, @@ -34,7 +28,7 @@ let stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip; optionsDoc = buildPackages.nixosOptionsDoc { - inherit options revision baseOptionsJSON warningsAreErrors allowDocBook; + inherit options revision baseOptionsJSON warningsAreErrors; transformOptions = opt: opt // { # Clean up declaration sites to not refer to the NixOS source tree. declarations = map stripAnyPrefixes opt.declarations; @@ -69,73 +63,6 @@ let optionIdPrefix = "test-opt-"; }; - toc = builtins.toFile "toc.xml" - '' - - - - - - - ''; - - manualXsltprocOptions = toString [ - "--param chapter.autolabel 0" - "--param part.autolabel 0" - "--param preface.autolabel 0" - "--param reference.autolabel 0" - "--param section.autolabel 0" - "--stringparam html.stylesheet 'style.css overrides.css highlightjs/mono-blue.css'" - "--stringparam html.script './highlightjs/highlight.pack.js ./highlightjs/loader.js'" - "--param xref.with.number.and.title 0" - "--param toc.section.depth 0" - "--param generate.consistent.ids 1" - "--stringparam admon.style ''" - "--stringparam callout.graphics.extension .svg" - "--stringparam current.docid manual" - "--param chunk.section.depth 0" - "--param chunk.first.sections 1" - "--param use.id.as.filename 1" - "--stringparam chunk.toc ${toc}" - ]; - - linterFunctions = '' - # outputs the context of an xmllint error output - # LEN lines around the failing line are printed - function context { - # length of context - local LEN=6 - # lines to print before error line - local BEFORE=4 - - # xmllint output lines are: - # file.xml:1234: there was an error on line 1234 - while IFS=':' read -r file line rest; do - echo - if [[ -n "$rest" ]]; then - echo "$file:$line:$rest" - local FROM=$(($line>$BEFORE ? $line - $BEFORE : 1)) - # number lines & filter context - nl --body-numbering=a "$file" | sed -n "$FROM,+$LEN p" - else - if [[ -n "$line" ]]; then - echo "$file:$line" - else - echo "$file" - fi - fi - done - } - - function lintrng { - xmllint --debug --noout --nonet \ - --relaxng ${docbook5}/xml/rng/docbook/docbook.rng \ - "$1" \ - 2>&1 | context 1>&2 - # ^ redirect assumes xmllint doesn’t print to stdout - } - ''; - prepareManualFromMD = '' cp -r --no-preserve=all $inputs/* . @@ -155,48 +82,6 @@ let ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json ''; - manual-combined = runCommand "nixos-manual-combined" - { inputs = lib.sourceFilesBySuffices ./. [ ".xml" ".md" ]; - nativeBuildInputs = [ pkgs.nixos-render-docs pkgs.libxml2.bin pkgs.libxslt.bin ]; - meta.description = "The NixOS manual as plain docbook XML"; - } - '' - ${prepareManualFromMD} - - nixos-render-docs -j $NIX_BUILD_CORES manual docbook \ - --manpage-urls ${manpageUrls} \ - --revision ${lib.escapeShellArg revision} \ - ./manual.md \ - ./manual-combined-pre.xml - - xsltproc \ - -o manual-combined.xml ${./../../lib/make-options-doc/postprocess-option-descriptions.xsl} \ - manual-combined-pre.xml - - ${linterFunctions} - - mkdir $out - cp manual-combined.xml $out/ - - lintrng $out/manual-combined.xml - ''; - - manpages-combined = runCommand "nixos-manpages-combined.xml" - { nativeBuildInputs = [ buildPackages.libxml2.bin buildPackages.libxslt.bin ]; - meta.description = "The NixOS manpages as plain docbook XML"; - } - '' - mkdir generated - cp -prd ${./man-pages.xml} man-pages.xml - ln -s ${optionsDoc.optionsDocBook} generated/options-db.xml - - xmllint --xinclude --noxincludenode --output $out ./man-pages.xml - - ${linterFunctions} - - lintrng $out - ''; - in rec { inherit (optionsDoc) optionsJSON optionsNix optionsDocBook; diff --git a/nixos/doc/manual/man-pages.xml b/nixos/doc/manual/man-pages.xml deleted file mode 100644 index 52183f1f9ee0..000000000000 --- a/nixos/doc/manual/man-pages.xml +++ /dev/null @@ -1,46 +0,0 @@ - - NixOS Reference Pages - - - EelcoDolstra - Author - - - The Nixpkgs/NixOS contributors - Author - - 2007-2022Eelco Dolstra and the Nixpkgs/NixOS contributors - - - - - configuration.nix - 5 - NixOS - - - - configuration.nix - NixOS system configuration specification - - - Description - - The file /etc/nixos/configuration.nix contains the - declarative specification of your NixOS system configuration. The command - nixos-rebuild takes this file and realises the system - configuration specified therein. - - - - Options - - You can use the following options in configuration.nix. - - - - - From 6fcb6eee774e3e5cc9b49f1b6a081ad5f992f5d2 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 13 Jun 2023 12:39:05 +0200 Subject: [PATCH 12/14] nixos/doc: set meta generator for html manuals properly we no longer have to look like docbook-xslt generates our manuals exclusively, so let's put something useful in there instead. --- nixos/doc/manual/default.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 03f0d26fdf93..40af4c1fa0b0 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -103,12 +103,10 @@ in rec { ${prepareManualFromMD} - # TODO generator is set like this because the docbook/md manual compare workflow will - # trigger if it's different nixos-render-docs -j $NIX_BUILD_CORES manual html \ --manpage-urls ${manpageUrls} \ --revision ${lib.escapeShellArg revision} \ - --generator "DocBook XSL Stylesheets V${docbook_xsl_ns.version}" \ + --generator "nixos-render-docs ${lib.version}" \ --stylesheet style.css \ --stylesheet overrides.css \ --stylesheet highlightjs/mono-blue.css \ From b9756b4de176bbf2086702050d24f8f90fd0f4d9 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 17:28:16 +0200 Subject: [PATCH 13/14] lib: unhide _module.args this was a temporary fix that should hopefully no longer be necessary. --- lib/modules.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 5821aeab20c7..9dcdb12bdcb1 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -134,11 +134,6 @@ let ${if prefix == [] then null # unset => visible else "internal"} = true; - # TODO: hidden during the markdown transition to not expose downstream - # users of the docs infra to markdown if they're not ready for it. - # we don't make this visible conditionally because it can impact - # performance (https://github.com/NixOS/nixpkgs/pull/208407#issuecomment-1368246192) - visible = false; # TODO: Change the type of this option to a submodule with a # freeformType, so that individual arguments can be documented # separately From 3e7649f01be8b7a4295bd414e6cb905affff7d66 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 11 Jun 2023 18:47:43 +0200 Subject: [PATCH 14/14] docbook-xsl: remove nixos-specific patch we're no longer using docbook to render the manpages, so we don't need this patch for our manuals to be usable. remove it to reduce the diff between nixos and other distros for this package. --- .../sgml+xml/stylesheets/xslt/docbook-xsl/default.nix | 6 +----- .../docbook-xsl/fix-man-options-duplication.patch | 11 ----------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/fix-man-options-duplication.patch diff --git a/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/default.nix b/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/default.nix index 2f9d22e57d89..735dfdb4f81b 100644 --- a/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/default.nix +++ b/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, substituteAll, fetchurl, fetchpatch, findXMLCatalogs, writeScriptBin, ruby, bash, withManOptDedupPatch ? false }: +{ lib, stdenv, substituteAll, fetchurl, fetchpatch, findXMLCatalogs, writeScriptBin, ruby, bash }: let @@ -36,10 +36,6 @@ let src = ./catalog-legacy-uris.patch; inherit legacySuffix suffix version; }) - ] ++ lib.optionals withManOptDedupPatch [ - # Fixes https://github.com/NixOS/nixpkgs/issues/166304 - # https://github.com/docbook/xslt10-stylesheets/pull/241 - ./fix-man-options-duplication.patch ]; propagatedBuildInputs = [ findXMLCatalogs ]; diff --git a/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/fix-man-options-duplication.patch b/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/fix-man-options-duplication.patch deleted file mode 100644 index 304d9781e6aa..000000000000 --- a/pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/fix-man-options-duplication.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/manpages/lists.xsl -+++ b/manpages/lists.xsl -@@ -110,7 +110,7 @@ - .RE - - -- -+ - - -