1
1
mirror of https://github.com/anoma/juvix.git synced 2024-09-11 16:26:33 +03:00
juvix/Makefile
Paul Cadman 2f4a3f809b
Run test suite in parallel (#2507)
## Overview

This PR makes the compiler pipeline thread-safe so that the test suite
can be run in parallel.

This is achieved by:
* Removing use of `{get, set, with}CurrentDir` functions.
* Adding locking around shared file resources like the the
global-project and internal build directory.

NB: **Locking is disabled for the main compiler target**, as it is
single threaded they are not required.

## Run test suite in parallel

To run the test suite in parallel you must add `--ta '+RTS -N -RTS'` to
your stack test arguments. For example:

```
stack test --fast --ta '+RTS -N -RTS'
```

The `-N` instructs the Haskell runtime to choose the number of threads
to use based on how many processors there are on your machine. You can
use `-Nn` to see the number of threads to `n`.

These flags are already [set in the
Makefile](e6dca22cfd/Makefile (L26))
when you or CI uses `stack test`.

## Locking

The Haskell package
[filelock](https://hackage.haskell.org/package/filelock) is used for
locking. File locks are used instead of MVars because Juvix code does
not control when new threads are created, they are created by the test
suite. This means that MVars created by Juvix code will have no effect,
because they are created independently on each test-suite thread.
Additionally the resources we're locking live on the filesystem and so
can be conveniently tagged by path.

### FileLock

The filelock library is wrapped in a FileLock effect:


e6dca22cfd/src/Juvix/Data/Effect/FileLock/Base.hs (L6-L8)

There is an [IO
interpreter](e6dca22cfd/src/Juvix/Data/Effect/FileLock/IO.hs (L8))
that uses filelock and an [no-op
interpreter](e6dca22cfd/src/Juvix/Data/Effect/FileLock/Permissive.hs (L7))
that just runs actions unconditionally.

### TaggedLock

To make the file locks simpler to use a TaggedLock effect is introduced:


e6dca22cfd/src/Juvix/Data/Effect/TaggedLock/Base.hs (L5-L11)

And convenience function:


e6dca22cfd/src/Juvix/Data/Effect/TaggedLock.hs (L28)

This allows an action to be locked, tagged by a directory that may or
may not exist. For example in the following code, an action is performed
on a directory `root` that may delete the directory before repopulating
the files. So the lockfile cannot be stored in the `root` itself.


e6dca22cfd/src/Juvix/Extra/Files.hs (L55-L60)

## Pipeline

As noted above, we only use locking in the test suite. The main app
target pipeline is single threaded and so locking is unnecessary. So the
interpretation of locks is parameterised so that locking can be disabled
e6dca22cfd/src/Juvix/Compiler/Pipeline/Run.hs (L64)
2023-11-16 16:19:52 +01:00

258 lines
6.0 KiB
Makefile

SHELL := /bin/bash
PWD=$(CURDIR)
PREFIX="$(PWD)/.stack-work/prefix"
UNAME := $(shell uname)
EXAMPLEMILESTONE=examples/milestone
EXAMPLEHTMLOUTPUT=docs/examples/html
EXAMPLES= Collatz/Collatz.juvix \
Fibonacci/Fibonacci.juvix \
Hanoi/Hanoi.juvix \
HelloWorld/HelloWorld.juvix \
PascalsTriangle/PascalsTriangle.juvix \
TicTacToe/CLI/TicTacToe.juvix \
Bank/Bank.juvix \
Tutorial/Tutorial.juvix
DEMO_EXAMPLE=examples/demo/Demo.juvix
MAKEAUXFLAGS?=-s
MAKE=make ${MAKEAUXFLAGS}
METAFILES:=README.md \
CHANGELOG.md \
CONTRIBUTING.md \
LICENSE.md
STACKFLAGS?=--jobs $(THREADS)
STACKTESTFLAGS?=--ta --hide-successes --ta --ansi-tricks=false --ta "+RTS -N -RTS"
SMOKEFLAGS?=--color --diff=git
STACK?=stack
JUVIXBIN?=juvix
ifeq ($(UNAME), Darwin)
THREADS := $(shell sysctl -n hw.logicalcpu)
else ifeq ($(UNAME), Linux)
THREADS := $(shell nproc)
else
THREADS := $(shell echo %NUMBER_OF_PROCESSORS%)
endif
all: build
clean: clean-runtime
@${STACK} clean --full
@rm -rf .hie
.PHONY: clean-hard
clean-hard: clean
@git clean -fdx
.PHONY: clean-runtime
clean-runtime: clean-juvix-build
@cd runtime && ${MAKE} clean
.PHONY: clean-juvix-build
clean-juvix-build:
@find . -type d -name '.juvix-build' | xargs rm -rf
repl:
@${STACK} ghci Juvix:lib ${STACKFLAGS}
# -- EXAMPLES HTML OUTPUT
.PHONY: html-examples
html-examples: $(EXAMPLES)
$(EXAMPLES):
$(eval OUTPUTDIR=$(EXAMPLEHTMLOUTPUT)/$(dir $@))
@mkdir -p ${OUTPUTDIR}
@${JUVIXBIN} html $(EXAMPLEMILESTONE)/$@ --output-dir=$(CURDIR)/${OUTPUTDIR}
.PHONY: demo-example
demo-example:
$(eval OUTPUTDIR=$(EXAMPLEHTMLOUTPUT)/Demo)
@mkdir -p ${OUTPUTDIR}
@${JUVIXBIN} html $(DEMO_EXAMPLE) --output-dir=$(CURDIR)/${OUTPUTDIR}
.PHONY : haddock
haddock :
@cabal --docdir=docs/ --htmldir=docs/ haddock --enable-documentation
# ------------------------------------------------------------------------------
# -- Codebase Health
# ------------------------------------------------------------------------------
ORMOLU?=${STACK} exec -- ormolu
ORMOLUFILES = $(shell git ls-files '*.hs' '*.hs-boot' | grep -v '^contrib/')
ORMOLUFLAGS?=--no-cabal
ORMOLUMODE?=inplace
.PHONY: ormolu
ormolu:
@${ORMOLU} ${ORMOLUFLAGS} \
--ghc-opt -XStandaloneDeriving \
--ghc-opt -XUnicodeSyntax \
--ghc-opt -XDerivingStrategies \
--ghc-opt -XMultiParamTypeClasses \
--ghc-opt -XTemplateHaskell \
--ghc-opt -XImportQualifiedPost \
--mode ${ORMOLUMODE} \
$(ORMOLUFILES)
.PHONY: format
format:
@${MAKE} clang-format
@${MAKE} ormolu
.PHONY: clang-format
clang-format:
@cd runtime && ${MAKE} format
JUVIX_PACKAGES_IN_REPO=$(shell find \
./examples \
./tests/positive \
./tests/negative \
-type d \( -name ".juvix-build" -o -name "FancyPaths" \) -prune -o \
-type f -name 'Package.juvix' -print \
| awk -F'/Package.juvix' '{print $$1}' | sort -u)
JUVIXFORMATFLAGS?=--in-place
JUVIXTYPECHECKFLAGS?=--only-errors
.PHONY: format-juvix-files
format-juvix-files:
@for p in $(JUVIX_PACKAGES_IN_REPO); do \
${JUVIXBIN} format $(JUVIXFORMATFLAGS) "$$p" > /dev/null 2>&1; \
exit_code=$$?; \
if [ $$exit_code -eq 0 ]; then \
echo "[OK] $$p is formatted"; \
elif [[ $$exit_code -ne 0 && "$$p" == *"tests/"* ]]; then \
echo "[CONTINUE] $$p is in tests directory."; \
else \
echo "[FAIL] $$p formatting failed" && exit 1; \
fi; \
done;
.PHONY: check-format-juvix-files
check-format-juvix-files:
@JUVIXFORMATFLAGS=--check ${MAKE} format-juvix-files
JUVIXEXAMPLEFILES=$(shell find ./examples \
-type d \( -name ".juvix-build" \) -prune -o \
-name "*.juvix" -print)
.PHONY: typecheck-juvix-examples
typecheck-juvix-examples:
@for file in $(JUVIXEXAMPLEFILES); do \
${JUVIXBIN} typecheck "$$file" $(JUVIXTYPECHECKFLAGS); \
exit_code=$$?; \
if [ $$exit_code -eq 0 ]; then \
echo "[OK] $$file typechecks"; \
else \
echo "[FAIL] Typecking failed for $$file" && exit 1; \
fi; \
done
.PHONY: check-ormolu
check-ormolu: export ORMOLUMODE = check
check-ormolu:
@${MAKE} ormolu
PRECOMMIT := $(shell command -v pre-commit 2> /dev/null)
.PHONY : install-pre-commit
install-pre-commit :
@$(if $(PRECOMMIT),, pip install pre-commit)
.PHONY : pre-commit
pre-commit :
@pre-commit run --all-files
# ------------------------------------------------------------------------------
# -- Build-Install-Test-Release
# ------------------------------------------------------------------------------
.PHONY: check-only
check-only:
@${MAKE} build \
&& ${MAKE} install \
&& ${MAKE} test \
&& ${MAKE} smoke \
&& ${MAKE} check-format-juvix-files \
&& ${MAKE} typecheck-juvix-examples \
&& ${MAKE} check-ormolu \
&& export SKIP=ormolu,format-juvix-examples,typecheck-juvix-examples \
&& ${MAKE} pre-commit
.PHONY: check
check: clean
@${MAKE} check-only
.PHONY : bench
bench: runtime submodules
@${STACK} bench ${STACKFLAGS}
# -- Build requirements
.PHONY: submodules
submodules:
@git submodule sync
@git submodule update --init --recursive
.PHONY: build
build: submodules runtime
@${STACK} build ${STACKFLAGS}
.PHONY: fast-build
fast-build: submodules runtime
@${STACK} build --fast ${STACKFLAGS}
.PHONY: runtime
runtime:
cd runtime && make -j 4 -s
# -- Install
.PHONY : install
install: runtime submodules
@${STACK} install ${STACKFLAGS}
.PHONY : fast-install
fast-install: runtime submodules
@${STACK} install --fast ${STACKFLAGS}
# -- Testing
.PHONY : test
test: build runtime submodules
@${STACK} test ${STACKFLAGS} ${STACKTESTFLAGS}
.PHONY : fast-test
fast-test: fast-build
@${STACK} test --fast ${STACKFLAGS} ${STACKTESTFLAGS}
.PHONY : test-skip-slow
test-skip-slow:
@${STACK} test ${STACKFLAGS} ${STACKTESTFLAGS} --ta '-p "! /slow tests/"'
.PHONY : fast-test-skip-slow
fast-test-skip-slow:
@${STACK} test --fast ${STACKFLAGS} ${STACKTESTFLAGS} --ta '-p "! /slow tests/"'
SMOKE := $(shell command -v smoke 2> /dev/null)
.PHONY : smoke-only
smoke-only:
@$(if $(SMOKE),, $(error "Smoke not found, please install it from https://github.com/jonaprieto/smoke"))
@smoke $(shell find tests -name '*.smoke.yaml')
.PHONY : smoke
smoke: install submodules
@${MAKE} smoke-only
# -- Release
.PHONY : changelog
changelog :
@github_changelog_generator