Merge branch 'main' into yorickvp/pkgs-fetchurl

.gitignore

@@ -5,4 +5,6 @@
 result
 interpreter
 __pycache__
-*.egg-info
+*.egg-info
+# symlink for devshell
+docs/src/reference

README.md

@@ -13,7 +13,9 @@
   <a href="https://github.com/nix-community/dream2nix/tree/main/examples/packages">Example Packages</a>
 </p>
 
-!!! Warning: dream2nix is unstable software. While simple UX is one of our main focus points, the APIs  are still under development. Do expect changes that will break your setup.
+!!! warning
+
+    dream2nix is unstable software. While simple UX is one of our main focus points, the APIs  are still under development. Do expect changes that will break your setup.
 
 ### legacy dream2nix
 

dev-flake/flake.lock

@@ -145,11 +145,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1718149104,
-        "narHash": "sha256-Ds1QpobBX2yoUDx9ZruqVGJ/uQPgcXoYuobBguyKEh8=",
+        "lastModified": 1719317636,
+        "narHash": "sha256-bu0xbu2Z6DDzA9LGV81yJunIti6r7tjUImeR8orAL/I=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e913ae340076bbb73d9f4d3d065c2bca7caafb16",
+        "rev": "9c513fc6fb75142f6aec6b7545cb8af2236b80f5",
         "type": "github"
       },
       "original": {

docs/README.md

@@ -0,0 +1,67 @@
+# About this Documentation
+
+Dream2nix documentation is generated from markdown via 
+[mkdocs](https://www.mkdocs.org/) and [mkdocs-material](https://squidfunk.github.io/mkdocs-material/).
+
+## Build
+
+You can build and server it locally with, i.e.:
+
+``` shellsession
+nix build .#website
+python3 -m http.server -d ./result
+```
+
+# Development shell
+
+Or alternatively run a development environment with:
+
+``` shellsession
+nix develop .#website
+```
+
+Upon entering the devshell, it will change into
+`./docs` and symlink a build of the [options reference](#options-reference)
+into `./docs/src/reference`. 
+
+Normal builds will always use an up-to-date options reference,
+but during development you need to update this symlink yourself
+and remove it after use. 
+
+i.e. from inside the shell in `./docs`:
+
+``` shellsession
+# update
+ln -sfT $(nix build --print-out-paths --no-link ..#optionsReference) ./src/reference
+# remove
+rm ./src/reference
+```
+
+## Options Reference
+
+The reference documentation for [modules](./modules.md) is auto-generated via
+a custom hook in `docs/hooks/render_options.py` and a derivation in
+`.#optionsReference`. 
+
+The derivation includes, for each module, an `options.json` file as generated by nix via `pkgs.nixosOptionsDoc` as well as a `README.md` file, copied from the modules source directory. The existence of such a `README.md` is used as an indicator on whether to include a module in the reference documentation.
+
+The hook runs whenever mkdocs renders one of the `README.md`s. Each of them gets concatenated with header and options reference, after the latter are run through jinja templates in `./docs/theme`.
+
+
+## Notes on Markdown
+
+Mkdocs uses a markdown dialect from [Python-Markdown](https://python-markdown.github.io/) with various, optional extensions listed in `./docs/mkdocs.yml`.
+
+This is different from the CommonMark dialect, as implemented by [markdown-it-py](https://pypi.org/project/markdown-it-py/) and used in NixOS official documentation.
+
+The differences between both don't seem to be too relevant for the markdown features used in our [options reference](#options-reference), but it's good
+to be aware of them when writing longer prose.
+
+[mkdocs-materials reference](https://squidfunk.github.io/mkdocs-material/reference) provides a
+good overview on useful extensions.
+
+## CI
+
+The documentation is published on GitHub pages via a GitHub action, defined in [.github/workflows/pages.yml](https://github.com/nix-community/dream2nix/blob/main/.github/workflows/pages.yml)
+
+    

docs/hooks/render_options.py

@@ -0,0 +1,122 @@
+import logging
+import json
+from pathlib import Path
+from collections import OrderedDict
+from typing import Dict, Tuple
+from urllib.request import UnknownHandler
+
+from mkdocs.structure.pages import Page
+from mkdocs.structure.nav import Navigation, Section
+from mkdocs.structure.files import Files
+from mkdocs.config.defaults import MkDocsConfig
+from mkdocs.plugins import event_priority
+
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.formatters import HtmlFormatter
+
+
+log = logging.getLogger("mkdocs")
+
+
+def is_reference_page(page: Page) -> bool:
+    return page.file.src_path.startswith("reference/")
+
+
+def pygments(code: str, lang: str) -> str:
+    return highlight(code, get_lexer_by_name(lang), HtmlFormatter())
+
+
+def sort_options(item: Tuple[str, Dict], module_name: str) -> int:
+    """
+    Sort the modules. First the one the page is about,
+    then single options, then the rest, alphabetically
+    """
+    name, option = item
+    if name == module_name:
+        return -1
+    elif len(option["children"]) == 0:
+        return 0
+    else:
+        return ord(name[0])
+
+
+def preprocess_options(options, module_name):
+    tree = dict()
+    for name, option in options.items():
+        if name.startswith("_module"):
+            continue
+        cursor = tree
+        parts = name.split(".")
+        for index, part in enumerate(parts):
+            if part not in cursor:
+                if index + 1 == len(parts):
+                    cursor[part] = dict(**option, children={})
+                else:
+                    cursor[part] = dict(children=dict())
+                    cursor = cursor[part]
+            else:
+                cursor = cursor[part]["children"]
+    return OrderedDict(sorted(tree.items(), key=lambda i: sort_options(i, module_name)))
+
+
+def on_page_markdown(
+    markdown: str, page: Page, config: MkDocsConfig, files: Files
+) -> str | None:
+    """Check whether the source path starts with "reference/".
+    If it does:
+    - render a header template, containing values from the source markdown file
+    - render the source markdown file
+    - render an options.json file containing nixos option definitions from the
+      same directory where the source file is found. Then render those options.
+    """
+    if not is_reference_page(page):
+        return markdown
+    src_path = Path(config.docs_dir) / page.file.src_path
+    module_name = src_path.parent.stem
+    env = config.theme.get_env()
+    env.filters["pygments"] = pygments
+
+    header = env.get_template("reference_header.html").render(meta=page.meta)
+
+    options_path = src_path.parent / "options.json"
+    if not options_path.exists():
+        log.error(f"{options_path} does not exist")
+        return None
+    with open(options_path, "r") as f:
+        options = preprocess_options(json.load(f), module_name)
+    reference = env.get_template("reference_options.html").render(options=options)
+
+    return "\n\n".join([header, markdown, reference])
+
+
+@event_priority(-100)
+def on_nav(nav: Navigation, config: MkDocsConfig, files: Files) -> Navigation | None:
+    """Customize the navigation: If a reference section is found,
+    filter for a "state" variable defined in a markdown files front-matter.
+    Leave all items where "state" equals "released" as-is, but put
+    all others in an "experimental" section below that."""
+    try:
+        reference_section = next(filter(lambda i: i.title == "Reference", nav.items))
+        reference_index = nav.items.index(reference_section)
+    except StopIteration:
+        # Return the navigation as-is if we don't find
+        # a reference section
+        return nav
+
+    released = []
+    experimental = []
+    for page in reference_section.children:
+        # to have metadata from the yaml front-matter available
+        page.read_source(config)
+        state = page.meta.get("state")
+        if state == "released":
+            released.append(page)
+        else:
+            experimental.append(page)
+
+    experimental_section = Section("Experimental Modules", experimental)
+    reference_section.children = released + [experimental_section]
+
+    nav.items[reference_index] = reference_section
+    return nav

docs/mkdocs.yml

@@ -0,0 +1,65 @@
+site_name: Dream2Nix
+docs_dir: "src"
+site_url: "https://nix-community.github.io/dream2nix"
+site_dir: !ENV out
+
+repo_url: https://github.com/nix-community/dream2nix
+repo_name: nix-community/dream2nix
+edit_uri: edit/main/docs/src/
+
+plugins:
+  - search
+  - awesome-pages
+
+hooks:
+  - hooks/render_options.py
+
+extra_css:
+  - style.css
+
+markdown_extensions:
+  - toc:
+      permalink: true
+  - tables
+  - admonition
+  - pymdownx.escapeall
+  - pymdownx.highlight:
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
+  - pymdownx.tasklist
+  - pymdownx.details
+
+validation:
+  omitted_files: warn
+  absolute_links: warn
+  unrecognized_links: warn
+
+theme:
+  custom_dir: theme
+  name: material
+  favicon: favicon.png
+  logo: favicon.png
+  features:
+    - search.suggest
+    - search.highlight
+    - instant
+    - navigation.instant
+    - navigation.instant.prefetch
+    - navigation.instant.progress
+    - navigation.tracking
+    - navigation.path
+    - navigation.top
+    - toc.follow
+    - content.code.copy
+    - content.code.annotate
+
+extra:
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/nix-community/dream2nix
+      name: Dream2nix on Github
+    - icon: fontawesome/solid/comments
+      link: https://matrix.to/#/#dream2nix:nixos.org
+      name: Dream2nix Matrix Channel

docs/src/.pages

@@ -0,0 +1,15 @@
+nav:
+  - "Home": index.md
+  - "Getting Started": gettingstarted.md
+  - Concepts:
+      - "Modules": modules.md
+      - "Overriding Dependencies": overrides.md
+  - Reference: reference
+  - Examples:
+      - "👉 example repository": https://github.com/nix-community/dream2nix/tree/main/examples/repo-with-packages
+      - "👉 example repository using flakes": https://github.com/nix-community/dream2nix/tree/main/examples/repo-with-packages-flake)
+      - "👉 example packages": https://github.com/nix-community/dream2nix/tree/main/examples/packages)
+  - Documentation: docs.md
+  - Notes:
+    - "v1 API design docs": v1-api
+    - development-roundups

docs/src/docs.md

@@ -0,0 +1 @@
+../README.md

docs/src/gettingstarted.md

@@ -1,5 +1,3 @@
-## Getting started
-
 To package a piece of software with dream2nix, you'd typically start with
 a bare-bones dream2nix flake like this:
 
@@ -60,8 +58,8 @@ And a `default.nix` that looks like this:
 ```
 
 To find out which dream2nix modules to import, browse through the modules
-on the left and the [examples](./examples.md). When getting started, the
-'/packages/languages' collection will be most helpful.
+on the left and the [examples](https://github.com/nix-community/dream2nix/tree/main/examples/packages). When getting started, the
+`./packages/languages` collection will be most helpful.
 
 Once you have imported a module, this module will make ecosystem-dependent
 functions available to create your package definition, such as `mkDerivation`

docs/src/modules.md

@@ -6,9 +6,9 @@ For some more background information, check out the initial exploration of this
 
 @edolstra 's [talk about this topic](https://www.youtube.com/watch?v=dTd499Y31ig) is also worth watching.
 
-# Benefits
+## Benefits
 
-## Deprecate override functions
+### Deprecate override functions
 
 Changing options of packages in nixpkgs can require chaining different override functions like this:
 
@@ -39,7 +39,7 @@ Changing options of packages in nixpkgs can require chaining different override
 
 See htop module definition [here](https://github.com/nix-community/dream2nix/blob/main/examples/packages/basics/htop-with-flags/default.nix).
 
-## Type safety
+### Type safety
 
 The following code in nixpkgs mkDerivation mysteriously skips the patches:
 
@@ -56,7 +56,7 @@ mkDerivation {
 A definition for option `[...].dontPatch' is not of type `boolean' [...]
 ```
 
-## Catch typos
+### Catch typos
 
 The following code in nixpkgs mkDerivation builds **without** openssl_3.
 
@@ -73,7 +73,7 @@ mkDerivation {
 The option `[...].nativBuildInputs' does not exist
 ```
 
-## Environment variables clearly defined
+### Environment variables clearly defined
 
 `dream2nix` requires a clear distinction between known parameters and user-defined variables.
 Defining `SOME_VARIABLE` at the top-level, would raise:
@@ -93,7 +93,7 @@ Instead it has to be defined under `env.`:
 }
 ```
 
-## Documentation / Discoverability
+### Documentation / Discoverability
 
 No more digging the source code to find possible options to override.
 
@@ -101,11 +101,11 @@ Documentation similar to [search.nixos.org](https://search.nixos.org) can be gen
 
 Every package built with `dream2nix` has a `.docs` attribute that builds an html documentation describing it's options.
 
-## Package blueprints
+### Package blueprints
 
 With `dream2nix`, packages don't need to be fully declared. Options can be left without defaults, requiring the consumer to complete the definition.
 
-## Flexibility
+### Flexibility
 
 The nixos module system gives maintainers more freedom over how packages are split into modules. Separation of concerns can be implemented more easily.
 For example, the dependency tree of a package set can be factored out into a separate module, allowing for simpler modification.

docs/src/style.css

@@ -0,0 +1,12 @@
+.md-header .md-logo {
+    background-color: white;
+    border-radius: 100%;
+}
+
+[data-md-component="toc"] .md-nav__link {
+    display: inline-flex;
+}
+
+.md-typeset td pre {
+    margin: 0;
+}

docs/src/v1-api/.pages

@@ -0,0 +1,3 @@
+nav:
+    - summary.md
+    - ...

docs/theme/partials/toc-item.html

@@ -0,0 +1,56 @@
+<!--
+     Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>
+
+     Permission is hereby granted, free of charge, to any person obtaining a copy
+     of this software and associated documentation files (the "Software"), to
+     deal in the Software without restriction, including without limitation the
+     rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+     sell copies of the Software, and to permit persons to whom the Software is
+     furnished to do so, subject to the following conditions:
+
+     The above copyright notice and this permission notice shall be included in
+     all copies or substantial portions of the Software.
+
+     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+     FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+     IN THE SOFTWARE.
+-->
+
+<!-- Table of contents item -->
+<li class="md-nav__item">
+    {%- set should_collapse = toc_item.level > 2 and toc_item.children -%}
+    {% if should_collapse -%}
+    <details>
+        <summary>
+            {%- endif %}
+            <a href="{{ toc_item.url }}" class="md-nav__link">
+                <span class="md-ellipsis">
+                    {%- if page.url.split("/")[-3] == "reference" -%}
+                    {{ toc_item.title.split(".")[-1] }}
+                    {%- else -%}
+                    {{ toc_item.title }}
+                    {%- endif -%}
+                </span>
+            </a>
+         {% if should_collapse -%}
+        </summary>
+        {%- endif %}
+
+        <!-- Table of contents list -->
+        {% if toc_item.children %}
+        <nav class="md-nav" aria-label="{{ toc_item.title | striptags }}">
+            <ul class="md-nav__list">
+                {% for toc_item in toc_item.children %}
+                {% include "partials/toc-item.html" %}
+                {% endfor %}
+            </ul>
+        </nav>
+        {% endif %}
+        {% if should_collapse -%}
+    </details>
+    {%- endif %}
+</li>

docs/theme/reference_header.html

@@ -0,0 +1,18 @@
+<table style="float: right;">
+    <tr>
+        <td>state:</td>
+        <td>{{ meta.state | default("experimental") }}</td>
+    </tr>
+    <tr>
+        <td>maintainers:</td>
+        <td>
+            {% if meta.maintainers -%}
+            {% for m in meta.maintainers -%}<a href="https://github.com/{{m}}">@{{m}}</a>{{ ", " if not loop.last else "" }}{%- endfor -%}
+            {%- else -%}
+            noone, maybe <strong>you</strong>?
+            {% endif %}
+        </td>
+    </tr>
+</table>
+
+## Introduction

docs/theme/reference_options.html

@@ -0,0 +1,40 @@
+## Options
+{%- for name, option in options.items() recursive %}
+
+##{{loop.depth * "#"}} {{ ((option.loc | join (".")) or name).replace("<", "&lt;").replace(">", "&gt;") }}
+
+{% if (option.description or '') != "This option has no description." -%}
+{{ option.description or '' }}
+{%- endif %}
+
+{% if "type" in option %}
+<table>
+    <tr>
+        <td>type</td>
+        <td>{{ option.type}} {{ "(read only)" if option.readOnly else "" }}</td>
+    </tr>
+    <tr>
+        <td>source</td>
+        <td>{%- for d in option.declarations -%}<a href="{{d.url}}">{{d.name}}</a>{{ ", " if not loop.last else "" }}{%- endfor -%}</td>
+    </tr>
+    {%- if option.default -%}
+    <tr>
+        <td>default</td>
+        <td>{{(option.default | default({})).text | pygments("nix")}}</td>
+    </tr>
+    {%- endif -%}
+    {%- if option.example -%}
+    <tr>
+        <td>example</td>
+        <td>
+            {{(option.example | default({})).text | pygments("nix")}}
+        </td>
+    </tr>
+    {%- endif -%}
+
+</table>
+{% endif %}
+{%- if option.children -%}
+{{ loop(option.children.items()) }}
+{%- endif %}
+{%- endfor %}

examples/packages/languages/nodejs-packaging-devshell-only/default.nix

@@ -22,11 +22,11 @@
   version = "1.5.0";
 
   mkDerivation = {
-    src = config.deps.fetchFromGitHub {
-      owner = "DavHau";
-      repo = "cowsay";
-      rev = "package-lock-v3";
-      sha256 = "sha256-KuZkGWl5An78IFR5uT/2jVTXdm71oWB+p143svYVkqQ=";
+    src = builtins.fetchGit {
+      shallow = true;
+      url = "https://github.com/DavHau/cowsay";
+      ref = "package-lock-v3";
+      rev = "c89952cb75e3e54b8ca0033bd3499297610083c7";
     };
     # allow devshell to be built -> CI pipeline happy
     buildPhase = "mkdir $out";

examples/packages/languages/nodejs-packaging/default.nix

@@ -10,11 +10,11 @@
   ];
 
   mkDerivation = {
-    src = config.deps.fetchFromGitHub {
-      owner = "DavHau";
-      repo = "cowsay";
-      rev = "package-lock-v3";
-      sha256 = "sha256-KuZkGWl5An78IFR5uT/2jVTXdm71oWB+p143svYVkqQ=";
+    src = builtins.fetchGit {
+      shallow = true;
+      url = "https://github.com/DavHau/cowsay";
+      ref = "package-lock-v3";
+      rev = "c89952cb75e3e54b8ca0033bd3499297610083c7";
     };
   };
 

examples/repo-with-packages/packages/hello/default.nix

@@ -9,7 +9,7 @@
   ];
 
   name = "hello";
-  version = "2.12";
+  version = "2.12.1";
 
   deps = {nixpkgs, ...}: {
     inherit
@@ -21,7 +21,7 @@
   mkDerivation = {
     src = builtins.fetchTarball {
       url = "https://ftp.gnu.org/gnu/hello/hello-${config.version}.tar.gz";
-      sha256 = "sha256-4GQeKLIxoWfYiOraJub5RsHNVQBr2H+3bfPP22PegdU=";
+      sha256 = "0xw6cr5jgi1ir13q6apvrivwmmpr5j8vbymp0x6ll0kcv6366hnn";
     };
   };
 }

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1718149104,
-        "narHash": "sha256-Ds1QpobBX2yoUDx9ZruqVGJ/uQPgcXoYuobBguyKEh8=",
+        "lastModified": 1719317636,
+        "narHash": "sha256-bu0xbu2Z6DDzA9LGV81yJunIti6r7tjUImeR8orAL/I=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e913ae340076bbb73d9f4d3d065c2bca7caafb16",
+        "rev": "9c513fc6fb75142f6aec6b7545cb8af2236b80f5",
         "type": "github"
       },
       "original": {

modules/dream2nix/WIP-groups/README.md

@@ -1,3 +1,10 @@
+---
+title: "groups"
+state: experimental
+maintainers:
+  - DavHau
+---
+
 Module to deal with package sets (so called groups in dream2nix)
 
 ## Separate different kinds of dependencies

modules/dream2nix/WIP-haskell-cabal/README.md

@@ -0,0 +1,6 @@
+---
+title: "haskell-cabal"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/WIP-nodejs-builder-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-builder-v3"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/WIP-nodejs-builder-v3/tests/packages/nodejs-packaging-package-lock/default.nix

@@ -9,11 +9,11 @@
   ];
 
   mkDerivation = {
-    src = config.deps.fetchFromGitHub {
-      owner = "DavHau";
-      repo = "cowsay";
-      rev = "package-lock-v3";
-      sha256 = "sha256-KuZkGWl5An78IFR5uT/2jVTXdm71oWB+p143svYVkqQ=";
+    src = builtins.fetchGit {
+      shallow = true;
+      url = "https://github.com/DavHau/cowsay";
+      ref = "package-lock-v3";
+      rev = "c89952cb75e3e54b8ca0033bd3499297610083c7";
     };
   };
 

modules/dream2nix/WIP-python-pdm/README.md

@@ -0,0 +1,6 @@
+---
+title: "python-pdm"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/WIP-python-pyproject/README.md

@@ -0,0 +1,6 @@
+---
+title: "python-pyproject"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/WIP-spago/README.md

@@ -0,0 +1,6 @@
+---
+title: "spago"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/buildPythonPackage/README.md

@@ -0,0 +1,6 @@
+---
+title: "buildPythonPackage"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/buildPythonPackage/interface.nix

@@ -93,13 +93,15 @@ in {
       description = ''
         Whether the pyproject format should be used. When set to `true`,
         `pypaBuildHook` will be used, and you can add the required build dependencies
-        from `build-system.requires` to `build-system`. Note that the pyproject
-        format falls back to using `setuptools`, so you can use `pyproject = true`
-        even if the package only has a `setup.py`. When set to `false`, you can
-        use the existing hooks or provide your own logic to build the
-        package. This can be useful for packages that don't support the pyproject
-        format. When unset, the legacy `setuptools` hooks are used for backwards
-        compatibility.
+        from `build-system.requires` to `build-system`.
+
+        Note that the pyproject format falls back to using `setuptools`, so
+        you can use `pyproject = true` even if the package only has a `setup.py`.
+        When set to `false`, you can use the existing hooks or provide your own
+        logic to build the package. This can be useful for packages that don't
+        support the pyproject format.
+
+        When unset, the legacy `setuptools` hooks are used for backwards compatibility.
       '';
     };
 

modules/dream2nix/buildRustPackage/README.md

@@ -0,0 +1,6 @@
+---
+title: "buildRustPackage"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/builtins-derivation/README.md

@@ -0,0 +1,6 @@
+---
+title: "builtins-derivation"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/core/README.md

@@ -0,0 +1,6 @@
+---
+title: "core"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/mkDerivation/README.md

@@ -1 +1,8 @@
+---
+title: "mkDerivation"
+state: released
+maintainers:
+  - DavHau
+---
+
 A package module based on the mkDerivation builder from nixpkgs

modules/dream2nix/multi-derivation-package/README.md

@@ -0,0 +1,6 @@
+---
+title: "multi-derivation-package"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nixpkgs-overrides/README.md

@@ -0,0 +1,6 @@
+---
+title: "nixpkgs-overrides"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-devshell-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-devshell-v3"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-devshell/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-devshell"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-granular-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-granular-v3"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-granular/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-granular"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-node-modules-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-node-modules-v3"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-node-modules/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-node-modules"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-package-json-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-package-json-v3"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-package-json/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-package-json"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-package-lock-v3/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-package-lock-v3"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/nodejs-package-lock/README.md

@@ -0,0 +1,6 @@
+---
+title: "nodejs-package-lock"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/overrides/README.md

@@ -0,0 +1,6 @@
+---
+title: "overrides"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/package-func/README.md

@@ -0,0 +1,6 @@
+---
+title: "package-func"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/php-composer-lock/README.md

@@ -0,0 +1,6 @@
+---
+title: "php-composer-lock"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/php-granular/README.md

@@ -0,0 +1,6 @@
+---
+title: "php-granular"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/pip/README.md

@@ -0,0 +1,30 @@
+---
+title: "pip"
+state: released
+maintainers:
+  - phaer
+---
+
+A module to package python projects via [pip](https://pip.pypa.io/).
+
+Under the hood, it uses [./pkgs/fetchPipMetadata](https://github.com/nix-community/dream2nix/tree/main/pkgs/fetchPipMetadata) to
+run `pip install --dry-run --report [...]` with reproducible inputs and converts the resulting installation report into a dream2nix
+lock file.
+
+!!! note
+
+    Due to limitations in `pip`s cross-platform support, the resulting
+    lock-files are platform-specific!
+    We therefore recommend setting `paths.lockFile` to `lock.${system}.json`
+    for all projects where you use the pip module.
+
+    Check out the [pdm module](../WIP-python-pdm/index.md) if you need a solution that
+    allows locking for multiple platforms at once!
+
+During building, it uses this lock file to build each dependency as well as the top-level package in separate derivations
+while allowing overrides and further customization via [dream2nix module system](../../modules.md).
+
+## Getting started
+
+See [Build a python project with pip](../../guides/pip.md).
+

modules/dream2nix/python-editables/editable.py

@@ -1,14 +1,14 @@
-import os
-import sys
-import json
-import shutil
-import subprocess
 import configparser
 import importlib
+import json
+import os
+import shutil
+import subprocess
+import sys
 from contextlib import redirect_stdout
-from textwrap import dedent
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from textwrap import dedent
 
 import tomli
 
@@ -173,6 +173,15 @@ def needs_update(args, dream2nix_python_dir):
     return old_args != args
 
 
+def export_environment_vars(python_environment, bin_dir, site_dir, site_packages):
+    print(
+        f"""
+    export PYTHONPATH="{site_dir}:{python_environment / site_packages}:$PYTHONPATH"
+    export PATH="{bin_dir}:${python_environment}/bin:$PATH"
+        """
+    )
+
+
 def pretty_print_editables(editables, root_dir, root_name):
     if os.environ.get("D2N_QUIET"):
         return
@@ -218,6 +227,7 @@ if __name__ == "__main__":
         if dream2nix_python_dir.exists():
             shutil.rmtree(dream2nix_python_dir)
     else:
+        export_environment_vars(python_environment, bin_dir, site_dir, site_packages)
         pretty_print_editables(editables, root_dir, root_name)
         exit(0)
 
@@ -259,14 +269,8 @@ for index, path in enumerate(sys.path):
         """
         )
 
-    print(
-        f"""
-export PYTHONPATH="{site_dir}:{python_environment / site_packages}:$PYTHONPATH"
-export PATH="{bin_dir}:${python_environment}/bin:$PATH"
-    """
-    )
-
     with open(dream2nix_python_dir / "editable-args.json", "w") as f:
         json.dump(args, f, indent=2)
 
+    export_environment_vars(python_environment, bin_dir, site_dir, site_packages)
     pretty_print_editables(editables, root_dir, root_name)

modules/dream2nix/rust-cargo-lock/README.md

@@ -0,0 +1,6 @@
+---
+title: "rust-cargo-lock"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/dream2nix/rust-crane/README.md

@@ -0,0 +1,6 @@
+---
+title: "rust-crane"
+state: "released"
+maintainers:
+  - DavHau
+---

modules/flake-parts/reference-website/default.nix

@@ -9,39 +9,33 @@
     system,
     ...
   }: let
-    modules' = self.modules.dream2nix;
-    modules = lib.filterAttrs (name: _: ! lib.elem name excludes) modules';
     dream2nixRoot = ../../../.;
     dream2nix = import dream2nixRoot;
-    excludes = [
-      # NOT WORKING
-      # TODO: fix those
-      "core"
-      "ui"
-      "docs"
-      "assertions"
-      "nixpkgs-overrides"
-
-      # doesn't need to be rendered
-      "_template"
-    ];
-    public = lib.genAttrs [
-      "nodejs-granular-v3"
-      "nodejs-package-lock-v3"
-      "php-composer-lock"
-      "php-granular"
-      "pip"
-      "rust-cargo-lock"
-      "rust-crane"
-    ] (name: null);
-
-    # interface
-    sourcePathStr = toString dream2nix;
     baseUrl = "https://github.com/nix-community/dream2nix/blob/master";
-    specialArgs = {
-      inherit dream2nix;
-      packageSets.nixpkgs = pkgs;
+
+    getOptions = {modules}: let
+      options = lib.flip lib.mapAttrs modules (
+        name: module: let
+          evaluated = lib.evalModules {
+            specialArgs = {
+              inherit dream2nix;
+              packageSets.nixpkgs = pkgs;
+            };
+            modules = [module];
+          };
+        in
+          evaluated.options
+      );
+      docs = lib.flip lib.mapAttrs options (name: options:
+        pkgs.nixosOptionsDoc {
+          inherit options;
+          inherit transformOptions;
+          warningsAreErrors = false;
+        });
+    in {
+      inherit options docs;
     };
+
     transformOptions = opt:
       opt
       // {
@@ -49,7 +43,7 @@
           map
           (
             decl: let
-              subpath = lib.removePrefix sourcePathStr (toString decl);
+              subpath = lib.removePrefix (toString dream2nix) (toString decl);
             in {
               url = baseUrl + subpath;
               name = "dream2nix" + subpath;
@@ -57,144 +51,106 @@
           )
           opt.declarations;
       };
-    # 0 = no chapters, 1 = one level of chapters, 2 = two levels of chapters ...
-    chaptersNesting = 1;
-    # A tree where nodes are (sub)chapters and leafs are options.
-    # Nesting can be arbitrary
-    chaptersTree = {
-      "Modules" =
-        lib.filterAttrs (name: _: public ? ${name}) optionsTree;
-      "Modules (Internal + Experimental)" =
-        lib.filterAttrs (name: _: ! public ? ${name}) optionsTree;
-    };
-    optionsTree = lib.flip lib.mapAttrs modules (
-      name: module: let
-        evaluated = lib.evalModules {
-          inherit specialArgs;
-          modules = [module];
-        };
-      in
-        # lib.trace "Rendering Module ${name}"
-        (builtins.removeAttrs evaluated.options ["_module"])
-    );
 
-    # implementation
-    highlight-js = let
-      highlight-core = pkgs.fetchurl {
-        url = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js";
-        hash = "sha256-g3pvpbDHNrUrveKythkPMF2j/J7UFoHbUyFQcFe1yEY=";
-      };
-      highlight-nix = pkgs.fetchurl {
-        url = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/nix.min.js";
-        hash = "sha256-BLoZ+/OroDAxMsdZ4GFZtQfsg6ZJeLVNeBzN/82dYgk=";
-      };
-    in
-      pkgs.runCommand "highlight-js" {} ''
-        cat ${highlight-core} > $out
-        cat ${highlight-nix} >> $out
+    modules =
+      lib.filterAttrs (
+        name: _:
+          ! lib.elem name
+          [
+            # NOT WORKING
+            # TODO: fix those
+            "core"
+            "ui"
+            "docs"
+            "assertions"
+            "nixpkgs-overrides"
+            # doesn't need to be rendered
+            "_template"
+          ]
+      )
+      dream2nix.modules.dream2nix;
+
+    options = getOptions {
+      inherit modules;
+    };
+
+    optionsReference = let
+      publicModules =
+        lib.filterAttrs
+        (n: v: lib.pathExists (v + "/README.md"))
+        modules;
+      createReference = name: sourcePath: ''
+        target_dir="$out/${name}/"
+        mkdir -p "$target_dir"
+        ln -s ${sourcePath}/README.md "$target_dir/index.md"
+        ln -s ${options.docs.${name}.optionsJSON}/share/doc/nixos/options.json "$target_dir"
+        cat > "$target_dir/.pages" <<EOF
+        collapse_single_pages: true
+        nav:
+          - ...
+        EOF
       '';
-    highlight-style = pkgs.fetchurl {
-      url = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/vs.min.css";
-      hash = "sha256-E1kfafj5iO+Tw/04hxdSG+OnvczojOXK2K0iCEYfzSw=";
-    };
-    optionsToMdFile = options: let
-      docs = pkgs.nixosOptionsDoc {
-        inherit options;
-        inherit transformOptions;
-        warningsAreErrors = false;
-      };
     in
-      docs.optionsCommonMark;
-
-    mdFiles = nesting: chapters:
-      if nesting == 0
-      then lib.mapAttrs (name: optionsToMdFile) chapters
-      else lib.concatMapAttrs (name: mdFiles (nesting - 1)) chapters;
-
-    mdFilesDir = pkgs.runCommand "md-files-dir" {} ''
-      mkdir -p $out
-      cp -r ${lib.concatStringsSep "\n cp -r " (lib.mapAttrsToList (name: file: "${file} $out/${name}.md") (mdFiles chaptersNesting chaptersTree))}
-    '';
-
-    spacing = depth:
-      if depth == 0
-      then "- "
-      else "  " + spacing (depth - 1);
-
-    # returns "" for chapters with nesting or an md file for chapters with options
-    chapterUrl = nesting: name:
-      if nesting == 0
-      then "options/${name}.md"
-      else "";
-
-    renderChapters = depth: nesting: chapters:
-      lib.concatStringsSep "\n"
-      (lib.flip lib.mapAttrsToList chapters (
-        name: chapter:
-          "${spacing depth}[${name}](${chapterUrl nesting name})"
-          + lib.optionalString (nesting > 0) (renderChapters (depth + 1) (nesting - 1) chapter)
-      ));
-
-    summaryMdFile =
-      pkgs.writeText "summary.md"
-      (renderChapters 0 chaptersNesting chaptersTree);
-
-    mdBookSource = pkgs.runCommand "website-src" {} ''
-      mkdir -p $out/options
-      cp ${summaryMdFile} $out/SUMMARY.md
-      cp -r ${mdFilesDir}/* $out/options/
-
-      # add table of contents for each md file
-      for file in $out/options/*.md; do
-        name="$(basename "$file")"
-        name="''${name%.md}"
-        echo "# $name - options" > "$file.tmp"
-        echo '<!-- toc -->' | cat - "$file" >> "$file.tmp"
-        mv $file.tmp $file
-      done
-    '';
+      pkgs.runCommand "reference" {
+      } ''
+        ${lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs createReference publicModules))}
+      '';
 
     website =
       pkgs.runCommand "website" {
         nativeBuildInputs = [
-          pkgs.mdbook
-          pkgs.mdbook-linkcheck
-          # This inserts a table of contents at each '<!-- toc -->'
-          inputs.mdbook-toc.defaultPackage.${system}
+          pkgs.python3.pkgs.mkdocs
+          pkgs.python3.pkgs.mkdocs-material
+          self.packages.${system}.mkdocs-awesome-pages-plugin
+          optionsReference
         ];
       } ''
-        cp -rL --no-preserve=mode ${dream2nixRoot}/website/* ./
-        cp -r ${mdBookSource}/* src/
-
-        # insert highlight.js
-        cp ${highlight-js} ./src/highlight.js
-        cp ${highlight-style} ./src/highlight.css
-
-        # merge original and generated SUMMARY.md
-        cp ${dream2nixRoot}/website/src/SUMMARY.md SUMMARY.md.orig
-        {
-          while read ln; do
-            case "$ln" in
-              "# Modules Reference")
-                echo "# Modules Reference"
-                cat ${mdBookSource}/SUMMARY.md
-                ;;
-              *)
-                echo "$ln"
-                ;;
-            esac
-          done
-        } < SUMMARY.md.orig > src/SUMMARY.md
-
-        # insert icon
-        mkdir -p ./theme
-        cp ${../../../modules/dream2nix/core/docs/theme/favicon.png} ./theme/favicon.png
-
-        ${pkgs.mdbook}/bin/mdbook build --dest-dir out
-        mv out/html $out
+        cp -rL --no-preserve=mode  ${dream2nixRoot}/docs/* .
+        ln -sfT ${optionsReference} ./src/reference
+        mkdocs build
       '';
   in {
+    packages.mkdocs-awesome-pages-plugin = pkgs.callPackage ./mkdocs-awesome-pages-plugin.nix {
+      inherit
+        (pkgs.python3.pkgs)
+        buildPythonPackage
+        mkdocs
+        wcmatch
+        natsort
+        beautifulsoup4
+        mock-open
+        importlib-metadata
+        poetry-core
+        pytestCheckHook
+        pythonOlder
+        ;
+    };
+    packages.optionsReference = optionsReference;
     packages.website = website;
-    packages.docs-generated-mdbook-src = mdBookSource;
+    devShells.website = let
+      pythonWithDeps = pkgs.python3.withPackages (
+        ps: [
+          ps.ipython
+          ps.black
+          ps.pytest
+          ps.pytest-cov
+        ]
+      );
+    in
+      pkgs.mkShell {
+        inputsFrom = [self.packages.${system}.website];
+        packages = [
+          pythonWithDeps
+        ];
+
+        shellHook = ''
+          cd $PRJ_ROOT/docs
+          if [ ! -d src/reference ]; then
+            echo "linking .#reference to src/reference, you need to update this manually\
+            and remove it before a production build"
+            ln -sfT $(nix build ..#reference --no-link --print-out-paths) src/reference
+          fi
+        '';
+      };
   };
 }

modules/flake-parts/reference-website/mkdocs-awesome-pages-plugin.nix

@@ -0,0 +1,59 @@
+# Can be removed from dream2nix repo when
+# https://github.com/NixOS/nixpkgs/pull/320709
+# is in nixpkgs-unstable
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  poetry-core,
+  mkdocs,
+  wcmatch,
+  natsort,
+  pytestCheckHook,
+  beautifulsoup4,
+  mock-open,
+  importlib-metadata,
+  pythonOlder,
+}:
+buildPythonPackage rec {
+  pname = "mkdocs-awesome-pages-plugin";
+  version = "2.9.2";
+  pyproject = true;
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchFromGitHub {
+    owner = "lukasgeiter";
+    repo = "mkdocs-awesome-pages-plugin";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-pYyZ84eNrslxgLSBr3teQqmV7hA+LHwJ+Z99QgPdh6U=";
+  };
+
+  propagatedBuildInputs = [
+    mkdocs
+    wcmatch
+    natsort
+  ];
+
+  nativeBuildInputs = [poetry-core];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    beautifulsoup4
+    mock-open
+    importlib-metadata
+  ];
+
+  disabledTestPaths = [
+    # requires "generatedfiles" mkdocs plugin
+    "mkdocs_awesome_pages_plugin/tests/e2e/test_gen_files.py"
+  ];
+
+  meta = with lib; {
+    description = "An MkDocs plugin that simplifies configuring page titles and their order";
+    homepage = "https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin";
+    changelog = "https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin/blob/v${version}/CHANGELOG";
+    license = licenses.mit;
+    maintainers = with maintainers; [phaer];
+  };
+}

pkgs/fetchPipMetadata/filter-pypi-responses.py

@@ -85,6 +85,11 @@ Response format:
 """
 
 
+def responseheaders(flow: http.HTTPFlow) -> None:
+    if "/simple/" not in flow.request.url:
+        flow.response.stream = True
+
+
 def response(flow: http.HTTPFlow) -> None:
     if not "/simple/" in flow.request.url:
         return

website/README.md

@@ -1,10 +0,0 @@
-# Website
-
-This directory contains markdown pages under ./src which are rendered to an html website using the tool mdbook.
-
-The website can be built via:
-```shellSession
-$ nix build .#website
-```
-
-In addition to what's already in ./src, this derivation generates and inserts reference documentation pages. One for each dream2nix module in /modules/dream2nix.

website/book.toml

@@ -1,23 +0,0 @@
-[book]
-authors = ["David Hauer", "Various contributors"]
-language = "en"
-multilingual = false
-src = "src"
-title = "dream2nix"
-
-[output.html]
-git-repository-url = "https://github.com/nix-community/dream2nix"
-edit-url-template = "https://github.com/nix-community/dream2nix/edit/main/modules/dream2nix/{path}"
-no-section-label = true
-additional-css = [ "style.css" ]
-additional-js = [ "no-edit-options.js" ]
-
-[output.html.print]
-enable = false  # big single page; don't need that
-
-[output.linkcheck]
-follow-web-links = false # no Internet during the build
-
-[preprocessor.toc]
-command = "mdbook-toc"
-renderer = ["html"]

website/no-edit-options.js

@@ -1,7 +0,0 @@
-
-if (window.location.pathname.match(/options/)) {
-  var buttons = document.querySelector("#menu-bar > div.right-buttons")
-  if (buttons != null) {
-    buttons.style.display = "none"
-  }
-}

website/src/SUMMARY.md

@@ -1,17 +0,0 @@
-
-- [Introduction](./intro.md)
-- [Getting started](./gettingstarted.md)
-- [Contact](./contact.md)
-
-# Examples
-- [Examples](./examples.md)
-# Concepts
-- [Modules](./modules.md)
-- [Overriding Dependencies](./overrides.md)
-
-# Modules Reference
-
-# Development Roundups
-- [April - June 2022](./development-roundups/2022-april-june.md)
-- [July - September 2022](./development-roundups/2022-july-september.md)
-

website/src/contact.md

@@ -1,2 +0,0 @@
-If you run into any problems or would like to discuss the project,
-check out the [#dream2nix:nixos.org](https://matrix.to/#/#dream2nix:nixos.org) Matrix channel.

website/src/examples.md

@@ -1,7 +0,0 @@
-# Examples for dream2nix
-
-- [👉 example repository](https://github.com/nix-community/dream2nix/tree/main/examples/repo-with-packages)
-- [👉 example repository using flakes](https://github.com/nix-community/dream2nix/tree/main/examples/repo-with-packages-flake)
-- [👉 example packages](https://github.com/nix-community/dream2nix/tree/main/examples/packages)
-  - [👉 basics](https://github.com/nix-community/dream2nix/tree/main/examples/packages/basics)
-  - [👉 languages](https://github.com/nix-community/dream2nix/tree/main/examples/packages/languages)

website/src/funding.md

@@ -1,2 +0,0 @@
-This project was funded by [NLnet](https://nlnet.nl/), which is a great way to get your open source project funded. **Applications are still open, you can [apply today](https://nlnet.nl/propose)**.
-For more information about dream2nix' funding, please refer to the [funding section of the readme](https://github.com/nix-community/dream2nix#funding)

website/src/warning.md

@@ -1 +0,0 @@
-!!! Warning: dream2nix is unstable software. While simple UX is one of our main focus points, the APIs  are still under development. Do expect changes that will break your setup.

website/style.css

@@ -1,31 +0,0 @@
-h1.menu-title::before {
-  content: "";
-  display: inline-block;
-  background-image: url(./favicon.png);
-  background-repeat: no-repeat;
-  background-size: contain;
-  width: 1.8ex;
-  height: 1.8ex;
-  margin-right: 0.9ex;
-  vertical-align: middle;
-}
-.light {
-  --links: #058;
-  --sidebar-active: #0af;
-}
-.sidebar .sidebar-scrollbox {
-  /* We don't use numbers, so we can take a consistent amount of space */
-  padding: var(--page-padding) !important;
-}
-.hljs {
-  background-color: #f6f7f6;
-}
-@media (prefers-color-scheme: dark) {
-  .hljs {
-    background-color: #1d1f21;
-  }
-}
-/* prevents chapters without links to be grey */
-.chapter li {
-  color: #000;
-}