chia|tests|pre-commit: Add missing __init__.py files (#8758)

* chia|tests: Add missing `__init__.py` files

* check and create needed __init__.py files via pre-commit

* plotting|tests: Handle `mypy` issues introduced by the `main` rebase

* pre-commit: Run `build-init-files.py` more verbose

* tests: Ignore `__pycache__` folders in `build-init-files.py`

* Update tests/build-init-files.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>

* Update tests/build-init-files.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>

* tests: Add shebang, make executable, add description

To `tests/build-init-files.py`

Co-authored-by: Kyle Altendorf <sda@fstab.net>

.pre-commit-config.yaml

@@ -6,6 +6,14 @@ repos:
         entry: ./tests/build-workflows.py --fail-on-update
         language: python
         pass_filenames: false
+-   repo: local
+    hooks:
+    -   id: init_py_files
+        name: __init__.py files
+        entry: python3 tests/build-init-files.py -v --root .
+        language: python
+        pass_filenames: false
+        additional_dependencies: [click~=7.1]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:

chia/plotting/create_plots.py

@@ -42,10 +42,10 @@ class PlotKeys:
 class PlotKeysResolver:
     def __init__(
         self,
-        farmer_public_key: str,
-        alt_fingerprint: int,
-        pool_public_key: str,
-        pool_contract_address: str,
+        farmer_public_key: Optional[str],
+        alt_fingerprint: Optional[int],
+        pool_public_key: Optional[str],
+        pool_contract_address: Optional[str],
         root_path: Path,
         log: logging.Logger,
         connect_to_daemon=False,
@@ -128,10 +128,10 @@ class PlotKeysResolver:
 
 
 async def resolve_plot_keys(
-    farmer_public_key: str,
-    alt_fingerprint: int,
-    pool_public_key: str,
-    pool_contract_address: str,
+    farmer_public_key: Optional[str],
+    alt_fingerprint: Optional[int],
+    pool_public_key: Optional[str],
+    pool_contract_address: Optional[str],
     root_path: Path,
     log: logging.Logger,
     connect_to_daemon=False,

tests/build-init-files.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+# Create missing `__init__.py` files in the source code folders (in "chia/" and "tests/").
+#
+# They are required by the python interpreter to properly identify modules/packages so that tools like `mypy` or an IDE
+# can work with their full capabilities.
+#
+# See https://docs.python.org/3/tutorial/modules.html#packages.
+#
+# Note: This script is run in a `pre-commit` hook (which runs on CI) to make sure we don't miss out any folder.
+
+import logging
+import pathlib
+import sys
+
+import click
+
+
+log_levels = {
+    0: logging.ERROR,
+    1: logging.WARNING,
+    2: logging.INFO,
+}
+
+
+@click.command()
+@click.option(
+    "-r", "--root", "root_str", type=click.Path(dir_okay=True, file_okay=False, resolve_path=True), default="."
+)
+@click.option("-v", "--verbose", count=True, help=f"Increase verbosity up to {len(log_levels) - 1} times")
+def command(verbose, root_str):
+    logger = logging.getLogger()
+    log_level = log_levels.get(verbose, min(log_levels.values()))
+    logger.setLevel(log_level)
+    stream_handler = logging.StreamHandler()
+    logger.addHandler(stream_handler)
+
+    tree_roots = ["chia", "tests"]
+    failed = False
+    root = pathlib.Path(root_str).resolve()
+    directories = sorted(
+        path
+        for tree_root in tree_roots
+        for path in root.joinpath(tree_root).rglob("**/")
+        if "__pycache__" not in path.parts
+    )
+
+    for path in directories:
+        init_path = path.joinpath("__init__.py")
+        # This has plenty of race hazards. If it messes up,
+        # it will likely get caught the next time.
+        if init_path.is_file() and not init_path.is_symlink():
+            logger.info(f"Found   : {init_path}")
+            continue
+        elif not init_path.exists():
+            failed = True
+            init_path.touch()
+            logger.warning(f"Created : {init_path}")
+        else:
+            failed = True
+            logger.error(f"Fail    : present but not a regular file: {init_path}", file=sys.stderr)
+
+    if failed:
+        raise click.ClickException("At least one __init__.py created or not a regular file")
+
+
+command()  # pylint: disable=no-value-for-parameter

tests/build-workflows.py

@@ -41,7 +41,7 @@ def read_file(filename: Path) -> str:
 
 # Input file
 def workflow_yaml_template_text(os):
-    return read_file(Path(root_path / f"runner-templates/build-test-{os}"))
+    return read_file(Path(root_path / f"runner_templates/build-test-{os}"))
 
 
 # Output files
@@ -64,9 +64,9 @@ def transform_template(template_text, replacements):
 # Replace with update_config
 def generate_replacements(conf, dir):
     replacements = {
-        "INSTALL_TIMELORD": read_file(Path(root_path / "runner-templates/install-timelord.include.yml")).rstrip(),
+        "INSTALL_TIMELORD": read_file(Path(root_path / "runner_templates/install-timelord.include.yml")).rstrip(),
         "CHECKOUT_TEST_BLOCKS_AND_PLOTS": read_file(
-            Path(root_path / "runner-templates/checkout-test-plots.include.yml")
+            Path(root_path / "runner_templates/checkout-test-plots.include.yml")
         ).rstrip(),
         "TEST_DIR": "",
         "TEST_NAME": "",

tests/plotting/test_plot_manager.py

@@ -477,14 +477,14 @@ async def test_callback_event_raises(test_environment, event_to_raise: PlotRefre
     expected_result = PlotRefreshResult()
     # Load dir_1
     add_plot_directory(env.root_path, str(env.dir_1.path))
-    expected_result.loaded = env.dir_1.plot_info_list()
+    expected_result.loaded = env.dir_1.plot_info_list()  # type: ignore[assignment]
     expected_result.removed = []
     expected_result.processed = len(env.dir_1)
     expected_result.remaining = 0
     await env.refresh_tester.run(expected_result)
     # Load dir_2
     add_plot_directory(env.root_path, str(env.dir_2.path))
-    expected_result.loaded = env.dir_2.plot_info_list()
+    expected_result.loaded = env.dir_2.plot_info_list()  # type: ignore[assignment]
     expected_result.removed = []
     expected_result.processed = len(env.dir_1) + len(env.dir_2)
     expected_result.remaining = 0
@@ -504,7 +504,7 @@ async def test_callback_event_raises(test_environment, event_to_raise: PlotRefre
     assert len(env.refresh_tester.plot_manager.no_key_filenames) == 0
     # The next run without the valid callback should lead to re-loading of all plot
     env.refresh_tester.plot_manager.set_refresh_callback(default_callback)
-    expected_result.loaded = env.dir_1.plot_info_list() + env.dir_2.plot_info_list()
+    expected_result.loaded = env.dir_1.plot_info_list() + env.dir_2.plot_info_list()  # type: ignore[assignment]
     expected_result.removed = []
     expected_result.processed = len(env.dir_1) + len(env.dir_2)
     expected_result.remaining = 0