Run all CI tests through a Docker container

Also reworks some dependency handling

.github/workflows/run-builds.yml

@@ -13,9 +13,9 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up OCaml
-        uses: avsm/setup-ocaml@v2
+        uses: ocaml/setup-ocaml@v2
         with:
-          ocaml-compiler: 4.11.x
+          ocaml-compiler: 4.13.x
           dune-cache: true
 
       - name: Install external dependencies
@@ -29,7 +29,7 @@ jobs:
           ./french_law/python/setup_env.sh
       - name: Install OCaml dependencies
         run: |
-          make dependencies-ocaml-with-z3
+          OPAMCONFIRMLEVEL=unsafe-yes make dependencies-ocaml-with-z3
       - name: Make build
         run: |
           OCAMLRUNPARAM=b opam exec -- make build

.github/workflows/run-make-all.yml

@@ -15,22 +15,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: Re-initialize python dependencies
-        run: |
-          ./french_law/python/setup_env.sh
-      - name: Install dependencies
-        run: |
-          OPAMCONFIRMLEVEL=unsafe-yes opam exec -- make dependencies pygments
-      - name: Check promoted files
-        run: |
-          opam exec -- make check-promoted > promotion.out 2>&1 || touch bad-promote
-      - name: Make all
-        run: |
-          OCAMLRUNPARAM=b opam exec -- make all
-      - name: Forward result from promotion check
-        run: |
-          if [ -e bad-promote ]; then
-            echo "[ERROR] Some promoted files were not up-to-date";
-            cat promotion.out;
-            exit 1
-          fi
+      - name: Prepare container with all dependencies
+        run: git archive HEAD | docker build - --target dev-build-context
+      - name: Run builds, checks and tests
+        run: git archive HEAD | docker build -

Dockerfile

@@ -1,13 +1,54 @@
-FROM ocaml/opam:ubuntu-lts-ocaml-4.12
+# Stage 1: setup an opam switch with all dependencies installed
+# (only depends on the opam files)
+FROM ocamlpro/ocaml:4.14 AS dev-build-context
 
-RUN sudo apt-get update && sudo apt-get install -y \
-    man2html \
-    colordiff \
-    latexmk \
-    python3 \
-    python3-pip \
-    libgmp-dev \
-    npm \
-    nodejs
+# pandoc is not in alpine stable yet, install it manually with an explicit repository
+RUN sudo apk add pandoc --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
 
-RUN sudo pip3 install virtualenv
+RUN mkdir catala
+WORKDIR catala
+
+# Get only the opam files at this stage to allow caching
+ADD --chown=ocaml:ocaml *.opam ./
+
+# trigger the selection of catala dev tools in opam
+ENV OPAMVAR_cataladevmode=1
+ENV OPAMVAR_catalaz3mode=1
+
+# Get a switch with all the dependencies installed
+RUN opam --cli=2.1 switch create catala ocaml-system && \
+    opam --cli=2.1 pin . --no-action && \
+    opam --cli=2.1 install . --with-test --with-doc --depext-only && \
+    opam --cli=2.1 install . --with-test --with-doc --deps-only && \
+    opam clean
+# Note: just one `opam switch create .` command should be enough once opam 2.1.3 is released (opam#5047 ; opam#5185)
+
+
+# Stage 2: get the whole repo, run checks and builds
+FROM dev-build-context AS build-all
+
+# Get the full repo
+ADD --chown=ocaml:ocaml . .
+
+# Prepare extra local dependencies
+RUN opam exec -- make pygments dependencies-js
+RUN opam exec -- ./french_law/python/setup_env.sh
+
+# OCaml backtraces may be useful on failure
+ENV OCAMLRUNPARAM=b
+
+# Check promoted files (but delay failure)
+RUN opam exec -- make check-promoted > promotion.out 2>&1 || touch bad-promote
+
+# Check the build
+RUN opam exec -- make build
+
+# Check tests & all alt targets
+RUN OCAMLRUNPARAM=b opam exec -- make all
+
+# Forward results of promotion check
+RUN if [ -e bad-promote ]; then \
+  echo "[ERROR] Some promoted files were not up-to-date"; \
+  cat promotion.out; \
+  exit 1; \
+fi

Makefile

@@ -15,23 +15,25 @@ K := $(foreach exec,$(EXECUTABLES),\
         $(if $(shell which $(exec)),some string,$(warning [WARNING] No "$(exec)" executable found. \
 				Please install this executable for everything to work smoothly)))
 
+OPAM = opam --cli=2.1
+
 dependencies-ocaml:
-	opam install . ./doc/catala-dev-dependencies.opam --deps-only --with-doc --with-test --update-invariant
+	$(OPAM) pin . --no-action
+	OPAMVAR_cataladevmode=1 $(OPAM) install . --with-doc --with-test --update-invariant --depext-only
+	OPAMVAR_cataladevmode=1 $(OPAM) install . --with-doc --with-test --update-invariant --deps-only
 
 dependencies-ocaml-with-z3:
-	opam install . ./doc/catala-dev-dependencies.opam z3 --deps-only --with-doc --with-test --update-invariant
+	$(OPAM) pin . --no-action
+	OPAMVAR_cataladevmode=1 OPAMVAR_catalaz3mode=1 $(OPAM) install . --with-doc --with-test --update-invariant --depext-only
+	OPAMVAR_cataladevmode=1 OPAMVAR_catalaz3mode=1 $(OPAM) install . --with-doc --with-test --update-invariant --deps-only
 
 dependencies-js:
 	$(MAKE) -C $(FRENCH_LAW_JS_LIB_DIR) dependencies
 
-init-submodules:
-	git submodule update --init
-
-
 #> dependencies				: Install the Catala OCaml, JS and Git dependencies
-dependencies: dependencies-ocaml dependencies-js init-submodules
+dependencies: dependencies-ocaml dependencies-js
 
-dependencies-with-z3: dependencies-ocaml-with-z3 dependencies-js init-submodules
+dependencies-with-z3: dependencies-ocaml-with-z3 dependencies-js
 
 ##########################################
 # Catala compiler rules

catala.opam

@@ -17,6 +17,7 @@ license: "Apache-2.0"
 homepage: "https://github.com/CatalaLang/catala"
 bug-reports: "https://github.com/CatalaLang/catala/issues"
 depends: [
+  "ocamlfind" {!= "1.9.5"}
   "dune" {>= "2.8"}
   "ocaml" {>= "4.13.0"}
   "ANSITerminal" {>= "0.8.2"}
@@ -39,6 +40,11 @@ depends: [
   "cppo" {>= "1"}
   "alcotest" {with-test & >= "1.5.0"}
   "odoc" {with-doc}
+  "ocamlformat" {cataladevmode & = "0.21.0"}
+  "obelisk" {cataladevmode}
+  "conf-npm" {cataladevmode}
+  "conf-python-3-dev" {cataladevmode}
+  "z3" {catalaz3mode}
 ]
 depopts: ["z3"]
 conflicts: [
@@ -59,3 +65,9 @@ build: [
   ]
 ]
 dev-repo: "git+https://github.com/CatalaLang/catala.git"
+depexts: [
+  ["man2html" "colordiff" "latexmk" "python3-pip" "pandoc"]
+    {cataladevmode & os-family = "debian"}
+  ["cgit" "colordiff" "texlive" "py3-pip" "py3-pygments"]
+    {cataladevmode & os-distribution = "alpine"}
+]

clerk.opam

@@ -35,4 +35,7 @@ build: [
   ]
 ]
 dev-repo: "git+https://github.com/CatalaLang/catala.git"
-depexts: ["ninja-build"] {os-family = "debian"}
+depexts: [
+  ["ninja-build"] {os-family = "debian"}
+  ["samurai"] {os-distribution = "alpine"}
+]