CatalaLang/catala
1
1
Fork 0
mirror of https://github.com/CatalaLang/catala.git synced 2024-11-08 07:51:43 +03:00
Code Issues Projects Releases Wiki Activity

Refactor Python dependency handling

Browse Source 
* Use the newer "pyproject.toml" for syntax highlighters
* Use venv for everything (previously, the pygment lexers were installed
  globally, but more recent Python releases frown upon that. This will probably
  be more stable in the long run...)
* Use one venv at the root of the project (`_python_venv/`) for both
  pygments and the runtime
* Initialise this venv automatically from the Makefile (it should no longer be
  necessary to run scattered `set_up.sh` scripts with mysterious interactions)

On the downsides, though:
* Only tested with Python 3.11 at the moment
* Need to remember to activate the venv (`. _python_venv/bin/activate`) from the
  root for e.g. HTML literate output to work. A more manageable solution could
  be for Catala to provide the lexers on-the-fly when calling `pygmentize`.
This commit is contained in:
Louis Gesbert 2023-03-02 11:47:51 +01:00
parent 3b2019f0c3
commit cf2e80ba51
14 changed files with 110 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,5 +1,6 @@
_build/
_opam/
_python_venv/
dune-workspace*
result
*.install
@ -14,4 +15,4 @@ node_modules/
build.ninja
.envrc
.direnv
.direnv

3
Dockerfile
View File

@ -31,8 +31,7 @@ FROM dev-build-context
ADD --chown=ocaml:ocaml . .
# Prepare extra local dependencies
RUN opam exec -- make pygments dependencies-js
RUN opam exec -- ./french_law/python/setup_env.sh
RUN opam exec -- make dependencies-python dependencies-js pygments
# OCaml backtraces may be useful on failure
ENV OCAMLRUNPARAM=b

92
Makefile
View File

@ -12,7 +12,7 @@ export
# Dependencies
##########################################
EXECUTABLES = groff python3 colordiff node pygmentize node npm ninja pandoc
EXECUTABLES = groff python3 colordiff node node npm ninja pandoc
K := $(foreach exec,$(EXECUTABLES),\
$(if $(shell which $(exec)),some string,$(warning [WARNING] No "$(exec)" executable found. \
Please install this executable for everything to work smoothly)))
@ -32,10 +32,32 @@ dependencies-ocaml-with-z3:
dependencies-js:
$(MAKE) -C $(FRENCH_LAW_JS_LIB_DIR) dependencies
#> dependencies : Install the Catala OCaml, JS and Git dependencies
dependencies: dependencies-ocaml dependencies-js
PY_VENV_DIR = _python_venv
dependencies-with-z3: dependencies-ocaml-with-z3 dependencies-js
PY_VENV_ACTIVATE = . $(PY_VENV_DIR)/bin/activate;
# Rebuild when requirements change
$(PY_VENV_DIR): $(PY_VENV_DIR)/stamp
$(PY_VENV_DIR)/stamp: \
runtimes/python/catala/pyproject.toml \
syntax_highlighting/en/pygments/pyproject.toml \
syntax_highlighting/fr/pygments/pyproject.toml \
syntax_highlighting/pl/pygments/pyproject.toml
test -d $(PY_VENV_DIR) || python3 -m venv $(PY_VENV_DIR)
$(PY_VENV_ACTIVATE) python3 -m pip install -U \
-e runtimes/python/catala \
-e syntax_highlighting/en/pygments \
-e syntax_highlighting/fr/pygments \
-e syntax_highlighting/pl/pygments
touch $@
dependencies-python: $(PY_VENV_DIR)
#> dependencies : Install the Catala OCaml, JS and Git dependencies
dependencies: dependencies-ocaml dependencies-js dependencies-python
dependencies-with-z3: dependencies-ocaml-with-z3 dependencies-js dependencies-python
##########################################
# Catala compiler rules
@ -104,17 +126,8 @@ SYNTAX_HIGHLIGHTING_FR=${CURDIR}/syntax_highlighting/fr
SYNTAX_HIGHLIGHTING_EN=${CURDIR}/syntax_highlighting/en
SYNTAX_HIGHLIGHTING_PL=${CURDIR}/syntax_highlighting/pl
pygmentize_fr: $(SYNTAX_HIGHLIGHTING_FR)/set_up_pygments.sh
chmod +x $<
$<
pygmentize_en: $(SYNTAX_HIGHLIGHTING_EN)/set_up_pygments.sh
chmod +x $<
$<
pygmentize_pl: $(SYNTAX_HIGHLIGHTING_PL)/set_up_pygments.sh
chmod +x $<
$<
pygmentize_%: $(PY_VENV_DIR)
$(PY_VENV_ACTIVATE) python3 -m pip install syntax_highlighting/$*/pygments
#> pygments : Extends your pygmentize executable with Catala lexers
pygments: pygmentize_fr pygmentize_en pygmentize_pl
@ -163,33 +176,35 @@ TUTORIAL_EN_DIR=$(EXAMPLES_DIR)/tutorial_en
TUTORIEL_FR_DIR=$(EXAMPLES_DIR)/tutoriel_fr
POLISH_TAXES_DIR=$(EXAMPLES_DIR)/polish_taxes
literate_aides_logement: build
$(MAKE) -C $(AIDES_LOGEMENT_DIR) aides_logement.tex
$(MAKE) -C $(AIDES_LOGEMENT_DIR) aides_logement.html
MAKEP = $(PY_VENV_ACTIVATE) $(MAKE)
literate_aides_logement: build $(PY_VENV_DIR)
$(MAKEP) -C $(AIDES_LOGEMENT_DIR) aides_logement.tex
$(MAKEP) -C $(AIDES_LOGEMENT_DIR) aides_logement.html
literate_allocations_familiales: build
$(MAKE) -C $(ALLOCATIONS_FAMILIALES_DIR) allocations_familiales.tex
$(MAKE) -C $(ALLOCATIONS_FAMILIALES_DIR) allocations_familiales.html
$(MAKEP) -C $(ALLOCATIONS_FAMILIALES_DIR) allocations_familiales.tex
$(MAKEP) -C $(ALLOCATIONS_FAMILIALES_DIR) allocations_familiales.html
literate_code_general_impots: build
$(MAKE) -C $(CODE_GENERAL_IMPOTS_DIR) code_general_impots.tex
$(MAKE) -C $(CODE_GENERAL_IMPOTS_DIR) code_general_impots.html
$(MAKEP) -C $(CODE_GENERAL_IMPOTS_DIR) code_general_impots.tex
$(MAKEP) -C $(CODE_GENERAL_IMPOTS_DIR) code_general_impots.html
literate_us_tax_code: build
$(MAKE) -C $(US_TAX_CODE_DIR) us_tax_code.tex
$(MAKE) -C $(US_TAX_CODE_DIR) us_tax_code.html
$(MAKEP) -C $(US_TAX_CODE_DIR) us_tax_code.tex
$(MAKEP) -C $(US_TAX_CODE_DIR) us_tax_code.html
literate_tutorial_en: build
$(MAKE) -C $(TUTORIAL_EN_DIR) tutorial_en.tex
$(MAKE) -C $(TUTORIAL_EN_DIR) tutorial_en.html
$(MAKEP) -C $(TUTORIAL_EN_DIR) tutorial_en.tex
$(MAKEP) -C $(TUTORIAL_EN_DIR) tutorial_en.html
literate_tutoriel_fr: build
$(MAKE) -C $(TUTORIEL_FR_DIR) tutoriel_fr.tex
$(MAKE) -C $(TUTORIEL_FR_DIR) tutoriel_fr.html
$(MAKEP) -C $(TUTORIEL_FR_DIR) tutoriel_fr.tex
$(MAKEP) -C $(TUTORIEL_FR_DIR) tutoriel_fr.html
literate_polish_taxes: build
$(MAKE) -C $(POLISH_TAXES_DIR) polish_taxes.tex
$(MAKE) -C $(POLISH_TAXES_DIR) polish_taxes.html
$(MAKEP) -C $(POLISH_TAXES_DIR) polish_taxes.tex
$(MAKEP) -C $(POLISH_TAXES_DIR) polish_taxes.html
#> literate_examples : Builds the .tex and .html versions of the examples code. Needs pygments to be installed and patched with Catala.
literate_examples: literate_allocations_familiales literate_code_general_impots \
@ -269,11 +284,6 @@ FRENCH_LAW_LIBRARY_PYTHON = \
$(FRENCH_LAW_PYTHON_LIB_DIR)/src/allocations_familiales.py \
$(FRENCH_LAW_PYTHON_LIB_DIR)/src/aides_logement.py
PY_VIRTUALENV = $(FRENCH_LAW_PYTHON_LIB_DIR)/env/bin/activate
$(PY_VIRTUALENV):
@$(if $(wildcard $(PY_VIRTUALENV)),,$(error "Python virtualenv not initialised, you need to run $(FRENCH_LAW_PYTHON_LIB_DIR)/setup_env.sh"))
$(FRENCH_LAW_LIBRARY_PYTHON):
dune build $@
@ -282,15 +292,11 @@ generate_french_law_library_python:
dune build $(FRENCH_LAW_LIBRARY_PYTHON)
#> type_french_law_library_python : Types the French law library Python sources with mypy
type_french_law_library_python: $(PY_VIRTUALENV) \
generate_french_law_library_python
. $(PY_VIRTUALENV) ;\
$(MAKE) -C $(FRENCH_LAW_PYTHON_LIB_DIR) type
type_french_law_library_python: $(PY_VENV_DIR) generate_french_law_library_python
$(MAKEP) -C $(FRENCH_LAW_PYTHON_LIB_DIR) type
run_french_law_library_benchmark_python: $(PY_VIRTUALENV) \
type_french_law_library_python
. $(PY_VIRTUALENV) ;\
$(MAKE) -C $(FRENCH_LAW_PYTHON_LIB_DIR) bench
run_french_law_library_benchmark_python: $(PY_ENV_DIR) type_french_law_library_python
$(MAKEP) -C $(FRENCH_LAW_PYTHON_LIB_DIR) bench
##########################################
# High-level test and benchmarks commands

14
french_law/python/README.md
View File

@ -3,9 +3,17 @@
This folder contains a ready-to-use Python library featuring French public
algorithms coded up in Catala.
The Python version expected to run the Python code is above 3.6. For the commands
noted below to run, you are expected to setup a virtual Python environment with
`virtualenv` by running the `setup_env.sh` script.
The Python version expected to run the Python code is above 3.6. For the
commands noted below to run, you are expected to setup a virtual Python
environment: run `make dependencies-python` from the root of the Catala
repository.
Then activate the environment (needs to be done every time you open a new shell
session). From the root of the Catala repository, run the following command:
```
. _python_venv/bin/activate
```
## Organization

21
syntax_highlighting/README.md
View File

@ -40,19 +40,20 @@ You can now reload VSCode and check that you have syntax highlighting on any `.c
## Pygments
Pygments is a Python-based versatile lexer for various
programming languages. To use a version of Pygments
augmented with the Catala plugin, simply enter from the root of the repository
Pygments is a Python-based versatile lexer for various programming languages. To
use a version of Pygments augmented with the Catala plugin, simply enter from
the root of the repository
sudo make pygments
make pygments
This will execute the
script `syntax_highlighting/fr/pygments/set_up_pygments.sh`,
`syntax_highlighting/pl/pygments/set_up_pygments.sh` and
`syntax_highlighting/en/pygments/set_up_pygments.sh`.
This will setup a Python virtual environment ("venv"), and install the syntax
highlighting plugins that allow Pygments to handle Catala files. Those are
defined in `syntax_highlighting/XX/pygments/`.
The scripts patch your `pygmentize` executable, used for instance by the `minted` LaTeX package.
It will now point to the Catala-enabled version with the appropriate `catala_*` lexer.
Pygments is used for instance by the `minted` LaTeX package. To make sure it is
available, you need to "activate" the python venv each time using:
. _python_venv/bin/activate
## GNU gedit

11
syntax_highlighting/en/pygments/pyproject.toml Normal file
View File

@ -0,0 +1,11 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "catala_en_lexer"
version = "0.8"
dependencies = ["pygments"]
[project.entry-points."pygments.lexers"]
catala-en-lexer = "catala_en_lexer.lexer:CatalaEnLexer"

9
syntax_highlighting/en/pygments/setup.py
View File

@ -1,10 +1,3 @@
from setuptools import setup, find_packages
setup(
name='catala_en_lexer',
packages=find_packages(),
entry_points="""
[pygments.lexers]
catala_en_lexer = catala_en_lexer.lexer:CatalaEnLexer
""",
)
setup()

6
syntax_highlighting/en/set_up_pygments.sh
View File

@ -1,6 +0,0 @@
#! /usr/bin/env sh
set -eu
cd "$(dirname "$0")"
cd pygments && python3 setup.py develop --user

11
syntax_highlighting/fr/pygments/pyproject.toml Normal file
View File

@ -0,0 +1,11 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "catala_fr_lexer"
version = "0.8"
dependencies = ["pygments"]
[project.entry-points."pygments.lexers"]
catala-fr-lexer = "catala_fr_lexer.lexer:CatalaFrLexer"

9
syntax_highlighting/fr/pygments/setup.py
View File

@ -1,10 +1,3 @@
from setuptools import setup, find_packages
setup(
name='catala_fr_lexer',
packages=find_packages(),
entry_points="""
[pygments.lexers]
catala_fr_lexer = catala_fr_lexer.lexer:CatalaFrLexer
""",
)
setup()

6
syntax_highlighting/fr/set_up_pygments.sh
View File

@ -1,6 +0,0 @@
#! /usr/bin/env sh
set -ue
cd "$(dirname "$0")"
cd pygments && python3 setup.py develop --user

11
syntax_highlighting/pl/pygments/pyproject.toml Normal file
View File

@ -0,0 +1,11 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "catala_pl_lexer"
version = "0.8"
dependencies = ["pygments"]
[project.entry-points."pygments.lexers"]
catala-pl-lexer = "catala_pl_lexer.lexer:CatalaPlLexer"

9
syntax_highlighting/pl/pygments/setup.py
View File

@ -1,10 +1,3 @@
from setuptools import setup, find_packages
setup(
name='catala_pl_lexer',
packages=find_packages(),
entry_points="""
[pygments.lexers]
catala_pl_lexer = catala_pl_lexer.lexer:CatalaPlLexer
""",
)
setup()

6
syntax_highlighting/pl/set_up_pygments.sh
View File

@ -1,6 +0,0 @@
#! /usr/bin/env sh
set -eu
cd "$(dirname "$0")"
cd pygments && python3 setup.py develop --user