Proof-of-Concept for testing code samples

There are a number of code samples throughout nix.dev. How do we know
they still work? We don't!

This commit introduces a way for us to extract these code samples into files
and then run tests against them in CI. This will hopefully help us catch
regressions in future updates to nix, NixOS and/or this guide.

Additionally, I included a darwin specific nix-shell configuration that I
use personally on my M1 Mac to work on this repo. Might be useful for someone.

.github/workflows/ci.yml

@@ -21,3 +21,5 @@ jobs:
       run: nix-build
     - name: Linkcheck
       run: nix-shell --run "make linkcheck"
+    - name: Run code block tests
+      run: nix-shell --run "./run_code_block_tests.sh"

.gitignore

@@ -1,5 +1,8 @@
-result
+.direnv/
+.envrc
 build/
+extracted/
 requirements_frozen.txt
-
-
+result
+source/_ext/__pycache__/
+TODO.txt

run_code_block_tests.sh

@@ -0,0 +1 @@
+find extracted -name "test_*" -exec echo "running {}" \; -execdir {} \;

shell-darwin.nix

@@ -0,0 +1,30 @@
+# @zupo uses this file to work on nix.dev on his M1 Monterey
+
+let
+  nixpkgs = builtins.fetchTarball {
+    # https://status.nixos.org/ -> nixos-21.11 on 2022-04-06
+    url = "https://github.com/nixos/nixpkgs/archive/ccb90fb9e11459aeaf83cc28d5f8910816d90dd0.tar.gz";
+  };
+  pkgs = import nixpkgs {};
+  poetry2nix = import (fetchTarball {
+    # https://github.com/nix-community/poetry2nix/commits/master on 2022-04-06
+    url = "https://github.com/nix-community/poetry2nix/archive/99c79568352799af09edaeefc858d337e6d9c56f.tar.gz";
+  }) {
+    pkgs = pkgs;
+  };
+
+  env = poetry2nix.mkPoetryEnv {
+    pyproject = ./pyproject.toml;
+    poetrylock = ./poetry.lock;
+    editablePackageSources = {};
+  };
+in
+
+pkgs.mkShell {
+  name = "dev-shell";
+  buildInputs = [
+    env
+    pkgs.poetry
+	  ];
+
+}

source/_ext/extractable_code_block.py

@@ -0,0 +1,81 @@
+from docutils import nodes
+from docutils.nodes import Node
+from docutils.parsers.rst import Directive
+from docutils.parsers.rst import directives
+from sphinx.directives import optional_int
+from sphinx.directives.code import CodeBlock
+from sphinx.util import logging
+from sphinx.util.typing import OptionSpec
+from typing import List
+
+import os
+import stat
+
+logger = logging.getLogger(__name__)
+
+
+class ExtractableCodeBlock(CodeBlock):
+    """A custom directive to extract code blocks into files.
+
+    We want our code samples to be tested in CI. This class overrides the
+    default `code-block` to allow passing a second argument: the filename
+    to extract the code block to.
+
+    Out-of-the-box Sphinx behaviour:
+
+    ```python
+    foo = "bar"
+    ```
+
+    Additional extraction of a code block in `mydoc.md` file into
+    `./extracted/mydoc/foo.py` file:
+
+    ```python foo.py
+    foo = "bar"
+    ```
+    """
+
+    EXTRACT_DIR = "extracted"
+    optional_arguments = 2
+
+    def run(self) -> List[Node]:
+        # This is out-of-the-box usage of code-blocks, with a single
+        # argument: the programming language
+        # Don't do any extraction
+        if len(self.arguments) < 2:
+            return super(ExtractableCodeBlock, self).run()
+
+        location = self.state_machine.get_source_and_line(self.lineno)
+
+        path = os.path.join(
+            # top-level dir containing extracted code blocks
+            self.EXTRACT_DIR,
+            # subdir containing extracted code blocks from a single .md file
+            os.path.splitext(os.path.basename(self.state.document.current_source))[0],
+            # file name of the extracted code block
+            self.arguments[1],
+        )
+        logger.info(
+            f"Extracting code block into {path}",
+            location=location,
+        )
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w") as f:
+            f.write(self.block_text)
+
+        # make bash scripts executable
+        if path.endswith(".sh"):
+            st = os.stat(path)
+            os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+        return super(ExtractableCodeBlock, self).run()
+
+
+def setup(app):
+    app.add_directive("code-block", ExtractableCodeBlock, override=True)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

source/conf.py

@@ -27,14 +27,20 @@ from datetime import date
 # If your documentation needs a minimal Sphinx version, state it here.
 # needs_sphinx = '1.0'
 
+import os
+import sys
+
+sys.path.append(os.path.abspath("./_ext"))
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'myst_parser',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx_copybutton'
+    "myst_parser",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx_copybutton",
+    "extractable_code_block",
 ]
 
 # Add myst-specific extensions, see what is available on
@@ -53,23 +59,23 @@ copybutton_prompt_text = r"# |\$ "
 copybutton_prompt_is_regexp = True
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'nix.dev'
-author = u'Domen Kožar'
-copyright = '2016-' + str(date.today().year) + ', ' + author
+project = "nix.dev"
+author = "Domen Kožar"
+copyright = "2016-" + str(date.today().year) + ", " + author
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -114,7 +120,7 @@ exclude_patterns = []
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -136,7 +142,7 @@ html_theme_options = {
     "path_to_docs": "source",
     "use_repository_button": True,
     "use_issues_button": True,
-    "use_edit_page_button": True
+    "use_edit_page_button": True,
 }
 
 
@@ -180,12 +186,12 @@ html_theme_options = {
 
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
-    '**': [
-        'about.html',
-        'search-field.html',
-        'sbt-sidebar-nav.html',
-        'subscribe.html',
-        'sponsors.html',
+    "**": [
+        "about.html",
+        "search-field.html",
+        "sbt-sidebar-nav.html",
+        "subscribe.html",
+        "sponsors.html",
     ],
 }
 
@@ -242,13 +248,10 @@ html_sidebars = {
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #'preamble': '',
-
     # Latex figure (float) alignment
     #'figure_align': 'htbp',
 }
@@ -257,8 +260,13 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'nixpkgs-cookbook.tex', 'nixpkgs-cookbook Documentation',
-     u'Domen Kožar', 'manual'),
+    (
+        master_doc,
+        "nixpkgs-cookbook.tex",
+        "nixpkgs-cookbook Documentation",
+        "Domen Kožar",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -287,8 +295,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'nixpkgs-cookbook', 'nixpkgs-cookbook Documentation',
-     [author], 1)
+    (master_doc, "nixpkgs-cookbook", "nixpkgs-cookbook Documentation", [author], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -301,9 +308,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'nixpkgs-cookbook', 'nixpkgs-cookbook Documentation',
-     author, 'nixpkgs-cookbook', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "nixpkgs-cookbook",
+        "nixpkgs-cookbook Documentation",
+        author,
+        "nixpkgs-cookbook",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
@@ -365,7 +378,7 @@ epub_copyright = copyright
 # epub_post_files = []
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 # The depth of the table of contents in toc.ncx.
 # epub_tocdepth = 3
@@ -391,9 +404,9 @@ epub_exclude_files = ['search.html']
 
 linkcheck_ignore = [
     # It's a SPA
-    r'https://app.terraform.io',
+    r"https://app.terraform.io",
     # It's dynamic
-    r'https://matrix.to'
+    r"https://matrix.to",
 ]
 
 # Anchors are not present in HTML

source/tutorials/dev-environment.md

@@ -4,7 +4,9 @@ Let's build a Python web application using the Flask web framework as an exercis
 
 Create a new file called `default.nix`. This file is conventionally used for specifying packages. Add the code:
 
-```nix
+
+
+```{code-block} nix default.nix
 { pkgs ? import <nixpkgs> {} }:
 
 pkgs.python3Packages.buildPythonApplication {
@@ -17,7 +19,7 @@ pkgs.python3Packages.buildPythonApplication {
 
 You will also need a simple Flask app as `myapp.py`:
 
-```python
+```{code-block} python myapp.py
 #! /usr/bin/env python
 
 from flask import Flask
@@ -37,7 +39,7 @@ if __name__ == "__main__":
 
 And a `setup.py` script:
 
-```python
+```{code-block} python setup.py
 from setuptools import setup
 
 setup(
@@ -52,7 +54,7 @@ setup(
 
 Now build the package with:
 
-```bash
+```{code-block} bash test_nix_build.sh
 nix-build
 ```
 
@@ -62,9 +64,31 @@ You may notice we can run the application from the package like this: `./result/
 
 ```bash
 nix-shell default.nix
-python3 myapp.py
+python myapp.py
 ```
 
 In this context, Nix takes on the role that you would otherwise use pip or virtualenv for. Nix installs required dependencies and separates the environment from others on your system.
 
 You can check this Nix configuration into version control and share it with others to make sure you are all running the same software. This is a great way to prevent configuration drift between different team members & contributors, especially when a project has many dependencies.
+
+You can also run a bash script like this one in your CI to make sure your `default.nix` keeps working in the future.
+
+```{code-block} bash test_myapp.sh
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash
+set -euo pipefail
+
+# start myapp in background and save the process id
+python myapp.py >> /dev/null 2>&1 &
+pid=$!
+
+if [[ $(curl --retry 3 --retry-delay 1 --retry-connrefused http://127.0.0.1:5000) == "Hello, Nix!" ]]; then
+    echo "SUCCESS: myapp.py is serving the expected string"
+    kill $pid
+    exit 0
+else
+    echo "FAIL: myapp.py is not serving the expected string"
+    kill $pid
+    exit 1
+fi
+```