Proof-of-Concept for testing code samples

There are a number of code samples throughout nix.dev. How do we know
they still work? We don't!

This commit introduces a way for us to extract these code samples into files
and then run tests against them in CI. This will hopefully help us catch
regressions in future updates to nix, NixOS and/or this guide.

Additionally, I included a darwin specific nix-shell configuration that I
use personally on my M1 Mac to work on this repo. Might be useful for someone.

.github/workflows/ci.yml

@@ -21,3 +21,5 @@ jobs:
       run: nix-build
     - name: Linkcheck
       run: nix-shell --run "make linkcheck"
+    - name: Run code block tests
+      run: nix-shell --run "./run_code_block_tests.sh"

.gitignore

@@ -1,5 +1,8 @@
-result
+.direnv/
+.envrc
 build/
+extracted/
 requirements_frozen.txt
-
-
+result
+source/_ext/__pycache__/
+TODO.txt

run_code_block_tests.sh

@@ -0,0 +1 @@
+find extracted -name "test_*" -exec echo "running {}" \; -execdir {} \;

shell-darwin.nix

@@ -0,0 +1,30 @@
+# @zupo uses this file to work on nix.dev on his M1 Monterey
+
+let
+  nixpkgs = builtins.fetchTarball {
+    # https://status.nixos.org/ -> nixos-21.11 on 2022-04-06
+    url = "https://github.com/nixos/nixpkgs/archive/ccb90fb9e11459aeaf83cc28d5f8910816d90dd0.tar.gz";
+  };
+  pkgs = import nixpkgs {};
+  poetry2nix = import (fetchTarball {
+    # https://github.com/nix-community/poetry2nix/commits/master on 2022-04-06
+    url = "https://github.com/nix-community/poetry2nix/archive/99c79568352799af09edaeefc858d337e6d9c56f.tar.gz";
+  }) {
+    pkgs = pkgs;
+  };
+
+  env = poetry2nix.mkPoetryEnv {
+    pyproject = ./pyproject.toml;
+    poetrylock = ./poetry.lock;
+    editablePackageSources = {};
+  };
+in
+
+pkgs.mkShell {
+  name = "dev-shell";
+  buildInputs = [
+    env
+    pkgs.poetry
+	  ];
+
+}

source/_ext/extractable_code_block.py

@@ -0,0 +1,81 @@
+from docutils import nodes
+from docutils.nodes import Node
+from docutils.parsers.rst import Directive
+from docutils.parsers.rst import directives
+from sphinx.directives import optional_int
+from sphinx.directives.code import CodeBlock
+from sphinx.util import logging
+from sphinx.util.typing import OptionSpec
+from typing import List
+
+import os
+import stat
+
+logger = logging.getLogger(__name__)
+
+
+class ExtractableCodeBlock(CodeBlock):
+    """A custom directive to extract code blocks into files.
+
+    We want our code samples to be tested in CI. This class overrides the
+    default `code-block` to allow passing a second argument: the filename
+    to extract the code block to.
+
+    Out-of-the-box Sphinx behaviour:
+
+    ```python
+    foo = "bar"
+    ```
+
+    Additional extraction of a code block in `mydoc.md` file into
+    `./extracted/mydoc/foo.py` file:
+
+    ```python foo.py
+    foo = "bar"
+    ```
+    """
+
+    EXTRACT_DIR = "extracted"
+    optional_arguments = 2
+
+    def run(self) -> List[Node]:
+        # This is out-of-the-box usage of code-blocks, with a single
+        # argument: the programming language
+        # Don't do any extraction
+        if len(self.arguments) < 2:
+            return super(ExtractableCodeBlock, self).run()
+
+        location = self.state_machine.get_source_and_line(self.lineno)
+
+        path = os.path.join(
+            # top-level dir containing extracted code blocks
+            self.EXTRACT_DIR,
+            # subdir containing extracted code blocks from a single .md file
+            os.path.splitext(os.path.basename(self.state.document.current_source))[0],
+            # file name of the extracted code block
+            self.arguments[1],
+        )
+        logger.info(
+            f"Extracting code block into {path}",
+            location=location,
+        )
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w") as f:
+            f.write(self.block_text)
+
+        # make bash scripts executable
+        if path.endswith(".sh"):
+            st = os.stat(path)
+            os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+        return super(ExtractableCodeBlock, self).run()
+
+
+def setup(app):
+    app.add_directive("code-block", ExtractableCodeBlock, override=True)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

source/conf.py

@@ -20,26 +20,32 @@ from datetime import date
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
+
+import os
+import sys
+
+sys.path.append(os.path.abspath("./_ext"))
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'myst_parser',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx_copybutton'
+    "myst_parser",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx_copybutton",
+    "extractable_code_block",
 ]
 
 # Add myst-specific extensions, see what is available on
 # https://myst-parser.readthedocs.io/en/latest/syntax/optional.html
-myst_enable_extensions =  [
+myst_enable_extensions = [
     "colon_fence",
     "linkify",
     "tasklist",
@@ -53,23 +59,23 @@ copybutton_prompt_text = r"# |\$ "
 copybutton_prompt_is_regexp = True
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'nix.dev'
-author = u'Domen Kožar'
-copyright = '2016-' + str(date.today().year) + ', ' + author
+project = "nix.dev"
+author = "Domen Kožar"
+copyright = "2016-" + str(date.today().year) + ", " + author
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -89,9 +95,9 @@ language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -100,27 +106,27 @@ exclude_patterns = []
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
@@ -136,150 +142,152 @@ html_theme_options = {
     "path_to_docs": "source",
     "use_repository_button": True,
     "use_issues_button": True,
-    "use_edit_page_button": True
+    "use_edit_page_button": True,
 }
 
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.
-#html_title = 'nixpkgs-cookbook vrolling'
+# html_title = 'nixpkgs-cookbook vrolling'
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-#html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not None, a 'Last updated on:' timestamp is inserted at every page
 # bottom, using the given strftime format.
 # The empty string is equivalent to '%b %d, %Y'.
-#html_last_updated_fmt = None
+# html_last_updated_fmt = None
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
-    '**': [
-        'about.html',
-        'search-field.html',
-        'sbt-sidebar-nav.html',
-        'subscribe.html',
-        'sponsors.html',
-        ],
+    "**": [
+        "about.html",
+        "search-field.html",
+        "sbt-sidebar-nav.html",
+        "subscribe.html",
+        "sponsors.html",
+    ],
 }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Language to be used for generating the HTML full-text search index.
 # Sphinx supports the following languages:
 #   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
 #   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
-#html_search_language = 'en'
+# html_search_language = 'en'
 
 # A dictionary with options for the search language support, empty by default.
 # 'ja' uses this config value.
 # 'zh' user can custom change `jieba` dictionary path.
-#html_search_options = {'type': 'default'}
+# html_search_options = {'type': 'default'}
 
 # The name of a javascript file (relative to the configuration directory) that
 # implements a search results scorer. If empty, the default will be used.
-#html_search_scorer = 'scorer.js'
+# html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-#htmlhelp_basename = 'nixpkgs-'
+# htmlhelp_basename = 'nixpkgs-'
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-
-# Latex figure (float) alignment
-#'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+    # Latex figure (float) alignment
+    #'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'nixpkgs-cookbook.tex', 'nixpkgs-cookbook Documentation',
-     u'Domen Kožar', 'manual'),
+    (
+        master_doc,
+        "nixpkgs-cookbook.tex",
+        "nixpkgs-cookbook Documentation",
+        "Domen Kožar",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
@@ -287,12 +295,11 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'nixpkgs-cookbook', 'nixpkgs-cookbook Documentation',
-     [author], 1)
+    (master_doc, "nixpkgs-cookbook", "nixpkgs-cookbook Documentation", [author], 1)
 ]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -301,22 +308,28 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'nixpkgs-cookbook', 'nixpkgs-cookbook Documentation',
-     author, 'nixpkgs-cookbook', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "nixpkgs-cookbook",
+        "nixpkgs-cookbook Documentation",
+        author,
+        "nixpkgs-cookbook",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False
 
 
 # -- Options for Epub output ----------------------------------------------
@@ -328,72 +341,72 @@ epub_publisher = author
 epub_copyright = copyright
 
 # The basename for the epub file. It defaults to the project name.
-#epub_basename = project
+# epub_basename = project
 
 # The HTML theme for the epub output. Since the default themes are not
 # optimized for small screen space, using the same theme for HTML and epub
 # output is usually not wise. This defaults to 'epub', a theme designed to save
 # visual space.
-#epub_theme = 'epub'
+# epub_theme = 'epub'
 
 # The language of the text. It defaults to the language option
 # or 'en' if the language is not set.
-#epub_language = ''
+# epub_language = ''
 
 # The scheme of the identifier. Typical schemes are ISBN or URL.
-#epub_scheme = ''
+# epub_scheme = ''
 
 # The unique identifier of the text. This can be a ISBN number
 # or the project homepage.
-#epub_identifier = ''
+# epub_identifier = ''
 
 # A unique identification for the text.
-#epub_uid = ''
+# epub_uid = ''
 
 # A tuple containing the cover image and cover page html template filenames.
-#epub_cover = ()
+# epub_cover = ()
 
 # A sequence of (type, uri, title) tuples for the guide element of content.opf.
-#epub_guide = ()
+# epub_guide = ()
 
 # HTML files that should be inserted before the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_pre_files = []
+# epub_pre_files = []
 
 # HTML files that should be inserted after the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_post_files = []
+# epub_post_files = []
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 # The depth of the table of contents in toc.ncx.
-#epub_tocdepth = 3
+# epub_tocdepth = 3
 
 # Allow duplicate toc entries.
-#epub_tocdup = True
+# epub_tocdup = True
 
 # Choose between 'default' and 'includehidden'.
-#epub_tocscope = 'default'
+# epub_tocscope = 'default'
 
 # Fix unsupported image types using the Pillow.
-#epub_fix_images = False
+# epub_fix_images = False
 
 # Scale large images.
-#epub_max_image_width = 0
+# epub_max_image_width = 0
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#epub_show_urls = 'inline'
+# epub_show_urls = 'inline'
 
 # If false, no index is generated.
-#epub_use_index = True
+# epub_use_index = True
 
 
 linkcheck_ignore = [
     # It's a SPA
-    r'https://app.terraform.io',
+    r"https://app.terraform.io",
     # It's dynamic
-    r'https://matrix.to'
+    r"https://matrix.to",
 ]
 
 # Anchors are not present in HTML

source/tutorials/dev-environment.md

@@ -4,7 +4,9 @@ Let's build a Python web application using the Flask web framework as an exercis
 
 Create a new file called `default.nix`. This file is conventionally used for specifying packages. Add the code:
 
-```nix
+
+
+```{code-block} nix default.nix
 { pkgs ? import <nixpkgs> {} }:
 
 pkgs.python3Packages.buildPythonApplication {
@@ -17,7 +19,7 @@ pkgs.python3Packages.buildPythonApplication {
 
 You will also need a simple Flask app as `myapp.py`:
 
-```python
+```{code-block} python myapp.py
 #! /usr/bin/env python
 
 from flask import Flask
@@ -37,7 +39,7 @@ if __name__ == "__main__":
 
 And a `setup.py` script:
 
-```python
+```{code-block} python setup.py
 from setuptools import setup
 
 setup(
@@ -52,7 +54,7 @@ setup(
 
 Now build the package with:
 
-```bash
+```{code-block} bash test_nix_build.sh
 nix-build
 ```
 
@@ -62,9 +64,31 @@ You may notice we can run the application from the package like this: `./result/
 
 ```bash
 nix-shell default.nix
-python3 myapp.py
+python myapp.py
 ```
 
 In this context, Nix takes on the role that you would otherwise use pip or virtualenv for. Nix installs required dependencies and separates the environment from others on your system.
 
 You can check this Nix configuration into version control and share it with others to make sure you are all running the same software. This is a great way to prevent configuration drift between different team members & contributors, especially when a project has many dependencies.
+
+You can also run a bash script like this one in your CI to make sure your `default.nix` keeps working in the future.
+
+```{code-block} bash test_myapp.sh
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash
+set -euo pipefail
+
+# start myapp in background and save the process id
+python myapp.py >> /dev/null 2>&1 &
+pid=$!
+
+if [[ $(curl --retry 3 --retry-delay 1 --retry-connrefused http://127.0.0.1:5000) == "Hello, Nix!" ]]; then
+    echo "SUCCESS: myapp.py is serving the expected string"
+    kill $pid
+    exit 0
+else
+    echo "FAIL: myapp.py is not serving the expected string"
+    kill $pid
+    exit 1
+fi
+```