diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py index 0d656556557f..79f5b93be8af 100644 --- a/maintainers/scripts/pluginupdate.py +++ b/maintainers/scripts/pluginupdate.py @@ -42,8 +42,6 @@ LOG_LEVELS = { } log = logging.getLogger() -log.addHandler(logging.StreamHandler()) - def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): """Retry calling the decorated function using an exponential backoff. @@ -203,7 +201,6 @@ class Editor: name: str, root: Path, get_plugins: str, - generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None], default_in: Optional[Path] = None, default_out: Optional[Path] = None, deprecated: Optional[Path] = None, @@ -213,7 +210,6 @@ class Editor: self.name = name self.root = root self.get_plugins = get_plugins - self._generate_nix = generate_nix self.default_in = default_in or root.joinpath(f"{name}-plugin-names") self.default_out = default_out or root.joinpath("generated.nix") self.deprecated = deprecated or root.joinpath("deprecated.json") @@ -226,9 +222,9 @@ class Editor: def load_plugin_spec(self, plugin_file) -> List[PluginDesc]: return load_plugin_spec(plugin_file) - def generate_nix(self, plugins, outfile): + def generate_nix(self, plugins, outfile: str): '''Returns nothing for now, writes directly to outfile''' - self._generate_nix(plugins, outfile) + raise NotImplementedError() def get_update(self, input_file: str, outfile: str, proc: int): return get_update(input_file, outfile, proc, editor=self) @@ -237,9 +233,58 @@ class Editor: def attr_path(self): return self.name + "Plugins" + def get_drv_name(self, name: str): + return self.attr_path + "." + name + def rewrite_input(self, *args, **kwargs): return rewrite_input(*args, **kwargs) + def create_parser(self): + parser = argparse.ArgumentParser( + description=( + f"Updates nix derivations for {self.name} plugins" + f"By default from {self.default_in} to {self.default_out}" + ) + ) + parser.add_argument( + "--add", + dest="add_plugins", + default=[], + action="append", + help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo", + ) + parser.add_argument( + "--input-names", + "-i", + dest="input_file", + default=self.default_in, + help="A list of plugins in the form owner/repo", + ) + parser.add_argument( + "--out", + "-o", + dest="outfile", + default=self.default_out, + help="Filename to save generated nix code", + ) + parser.add_argument( + "--proc", + "-p", + dest="proc", + type=int, + default=30, + help="Number of concurrent processes to spawn.", + ) + parser.add_argument( + "--no-commit", "-n", action="store_true", default=False, + help="Whether to autocommit changes" + ) + parser.add_argument( + "--debug", "-d", choices=LOG_LEVELS.keys(), + default=logging.getLevelName(logging.WARN), + help="Adjust log level" + ) + return parser @@ -466,54 +511,6 @@ def rewrite_input( with open(input_file, "w") as f: f.writelines(lines) -# TODO move to Editor ? -def parse_args(editor: Editor): - parser = argparse.ArgumentParser( - description=( - f"Updates nix derivations for {editor.name} plugins" - f"By default from {editor.default_in} to {editor.default_out}" - ) - ) - parser.add_argument( - "--add", - dest="add_plugins", - default=[], - action="append", - help=f"Plugin to add to {editor.attr_path} from Github in the form owner/repo", - ) - parser.add_argument( - "--input-names", - "-i", - dest="input_file", - default=editor.default_in, - help="A list of plugins in the form owner/repo", - ) - parser.add_argument( - "--out", - "-o", - dest="outfile", - default=editor.default_out, - help="Filename to save generated nix code", - ) - parser.add_argument( - "--proc", - "-p", - dest="proc", - type=int, - default=30, - help="Number of concurrent processes to spawn.", - ) - parser.add_argument( - "--no-commit", "-n", action="store_true", default=False, - help="Whether to autocommit changes" - ) - parser.add_argument( - "--debug", "-d", choices=LOG_LEVELS.keys(), - default=logging.getLevelName(logging.WARN), - help="Adjust log level" - ) - return parser.parse_args() - def commit(repo: git.Repo, message: str, files: List[Path]) -> None: repo.index.add([str(f.resolve()) for f in files]) @@ -547,12 +544,10 @@ def get_update(input_file: str, outfile: str, proc: int, editor: Editor): return update -def update_plugins(editor: Editor): +def update_plugins(editor: Editor, args): """The main entry function of this module. All input arguments are grouped in the `Editor`.""" - args = parse_args(editor) log.setLevel(LOG_LEVELS[args.debug]) - log.info("Start updating plugins") nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True) update = editor.get_update(args.input_file, args.outfile, args.proc) @@ -581,7 +576,7 @@ def update_plugins(editor: Editor): if autocommit: commit( nixpkgs_repo, - "{editor.attr_path}.{name}: init at {version}".format( + "{editor.get_drv_name name}: init at {version}".format( editor=editor.name, name=plugin.normalized_name, version=plugin.version ), [args.outfile, args.input_file], diff --git a/maintainers/scripts/update-luarocks-packages b/maintainers/scripts/update-luarocks-packages index 2524b4c9b89e..5123f86e93d0 100755 --- a/maintainers/scripts/update-luarocks-packages +++ b/maintainers/scripts/update-luarocks-packages @@ -16,20 +16,17 @@ from dataclasses import dataclass import subprocess import csv import logging +import textwrap +from multiprocessing.dummy import Pool -from typing import List +from typing import List, Tuple from pathlib import Path -LOG_LEVELS = { - logging.getLevelName(level): level for level in [ - logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ] -} - log = logging.getLogger() log.addHandler(logging.StreamHandler()) ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent -from pluginupdate import Editor, parse_args, update_plugins, PluginDesc, CleanEnvironment +from pluginupdate import Editor, update_plugins, PluginDesc, CleanEnvironment, LOG_LEVELS, Cache PKG_LIST="maintainers/scripts/luarocks-packages.csv" TMP_FILE="$(mktemp)" @@ -67,12 +64,11 @@ class LuaEditor(Editor): def get_current_plugins(self): return [] - def load_plugin_spec(self, input_file) -> List[PluginDesc]: + def load_plugin_spec(self, input_file) -> List[LuaPlugin]: luaPackages = [] csvfilename=input_file log.info("Loading package descriptions from %s", csvfilename) - with open(csvfilename, newline='') as csvfile: reader = csv.DictReader(csvfile,) for row in reader: @@ -81,96 +77,115 @@ class LuaEditor(Editor): luaPackages.append(plugin) return luaPackages + def generate_nix( + self, + results: List[Tuple[LuaPlugin, str]], + outfilename: str + ): + + with tempfile.NamedTemporaryFile("w+") as f: + f.write(HEADER) + header2 = textwrap.dedent( + # header2 = inspect.cleandoc( + """ + { self, stdenv, lib, fetchurl, fetchgit, ... } @ args: + self: super: + with self; + { + """) + f.write(header2) + for (plugin, nix_expr) in results: + f.write(f"{plugin.normalized_name} = {nix_expr}") + f.write(FOOTER) + f.flush() + + # if everything went fine, move the generated file to its destination + # using copy since move doesn't work across disks + shutil.copy(f.name, outfilename) + + print(f"updated {outfilename}") + @property def attr_path(self): return "luaPackages" - def get_update(self, input_file: str, outfile: str, _: int): + def get_update(self, input_file: str, outfile: str, proc: int): + cache: Cache = Cache(self.cache_file) + _prefetch = generate_pkg_nix def update() -> dict: plugin_specs = self.load_plugin_spec(input_file) + sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower()) - self.generate_nix(plugin_specs, outfile) + try: + pool = Pool(processes=proc) + results = pool.map(_prefetch, sorted_plugin_specs) + finally: + pass + + self.generate_nix(results, outfile) redirects = [] return redirects return update - def rewrite_input(self, *args, **kwargs): + def rewrite_input(self, input_file: str, *args, **kwargs): + # vim plugin reads the file before update but that shouldn't be our case # not implemented yet + # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers'] + # input_file = "toto.csv" + # with open(input_file, newline='') as csvfile: + # writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + # writer.writeheader() + # for row in reader: + # # name,server,version,luaversion,maintainers + # plugin = LuaPlugin(**row) + # luaPackages.append(plugin) pass -def generate_nix( - plugins: List[LuaPlugin], - outfilename: str - ): - sorted_plugins = sorted(plugins, key=lambda v: v.name.lower()) +def generate_pkg_nix(plug: LuaPlugin): + ''' + Generate nix expression for a luarocks package + Our cache key associates "p.name-p.version" to its rockspec + ''' + log.debug("Generating nix expression for %s", plug.name) + cmd = [ "luarocks", "nix", plug.name] - # plug = {} - # selon le manifest luarocks.org/manifest - def _generate_pkg_nix(plug): - cmd = [ "luarocks", "nix", plug.name] - if plug.server: - cmd.append(f"--only-server={plug.server}") + if plug.server: + cmd.append(f"--only-server={plug.server}") - if plug.maintainers: - cmd.append(f"--maintainers={plug.maintainers}") + if plug.maintainers: + cmd.append(f"--maintainers={plug.maintainers}") - if plug.version: - cmd.append(plug.version) + if plug.version: + cmd.append(plug.version) - if plug.luaversion: - with CleanEnvironment(): - local_pkgs = str(ROOT.resolve()) - cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"] + if plug.luaversion: + with CleanEnvironment(): + local_pkgs = str(ROOT.resolve()) + cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"] - log.debug("running %s", cmd2) - lua_drv_path=subprocess.check_output(cmd2, text=True).strip() - cmd.append(f"--lua-dir={lua_drv_path}/bin") - - log.debug("running %s", cmd) - output = subprocess.check_output(cmd, text=True) - return output - - with tempfile.NamedTemporaryFile("w+") as f: - f.write(HEADER) - f.write(""" -{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args: -self: super: -with self; -{ -""") - - for plugin in sorted_plugins: - - nix_expr = _generate_pkg_nix(plugin) - f.write(f"{plugin.normalized_name} = {nix_expr}" - ) - f.write(FOOTER) - f.flush() - - # if everything went fine, move the generated file to its destination - # using copy since move doesn't work across disks - shutil.copy(f.name, outfilename) - - print(f"updated {outfilename}") - -def load_plugin_spec(): - pass + log.debug("running %s", ' '.join(cmd2)) + lua_drv_path=subprocess.check_output(cmd2, text=True).strip() + cmd.append(f"--lua-dir={lua_drv_path}/bin") + log.debug("running %s", cmd) + output = subprocess.check_output(cmd, text=True) + return (plug, output) def main(): - editor = LuaEditor("lua", ROOT, '', generate_nix, + editor = LuaEditor("lua", ROOT, '', default_in = ROOT.joinpath(PKG_LIST), default_out = ROOT.joinpath(GENERATED_NIXFILE) ) - args = parse_args(editor) + parser = editor.create_parser() + args = parser.parse_args() log.setLevel(LOG_LEVELS[args.debug]) - update_plugins(editor) + update_plugins(editor, args) if __name__ == "__main__": diff --git a/pkgs/applications/editors/kakoune/plugins/update.py b/pkgs/applications/editors/kakoune/plugins/update.py index b6a4bfe4f415..40a28d9afe2c 100755 --- a/pkgs/applications/editors/kakoune/plugins/update.py +++ b/pkgs/applications/editors/kakoune/plugins/update.py @@ -39,52 +39,57 @@ in lib.filterAttrs (n: v: v != null) checksums)""" HEADER = "# This file has been generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!" +class KakouneEditor(pluginupdate.Editor): -def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str): - sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) - with open(outfile, "w+") as f: - f.write(HEADER) - f.write( - """ -{ lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }: -let - packages = ( self: -{""" - ) - for owner, repo, plugin in sorted_plugins: - if plugin.has_submodules: - submodule_attr = "\n fetchSubmodules = true;" - else: - submodule_attr = "" + def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str): + sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) + with open(outfile, "w+") as f: + f.write(HEADER) f.write( - f""" - {plugin.normalized_name} = buildKakounePluginFrom2Nix {{ - pname = "{plugin.normalized_name}"; - version = "{plugin.version}"; - src = fetchFromGitHub {{ - owner = "{owner}"; - repo = "{repo}"; - rev = "{plugin.commit}"; - sha256 = "{plugin.sha256}";{submodule_attr} - }}; - meta.homepage = "https://github.com/{owner}/{repo}/"; - }}; -""" + """ + { lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }: + let + packages = ( self: + {""" ) - f.write( - """ -}); -in lib.fix' (lib.extends overrides packages) -""" - ) - print(f"updated {outfile}") + for owner, repo, plugin in sorted_plugins: + if plugin.has_submodules: + submodule_attr = "\n fetchSubmodules = true;" + else: + submodule_attr = "" + + f.write( + f""" + {plugin.normalized_name} = buildKakounePluginFrom2Nix {{ + pname = "{plugin.normalized_name}"; + version = "{plugin.version}"; + src = fetchFromGitHub {{ + owner = "{owner}"; + repo = "{repo}"; + rev = "{plugin.commit}"; + sha256 = "{plugin.sha256}";{submodule_attr} + }}; + meta.homepage = "https://github.com/{owner}/{repo}/"; + }}; + """ + ) + f.write( + """ + }); + in lib.fix' (lib.extends overrides packages) + """ + ) + print(f"updated {outfile}") def main(): - editor = pluginupdate.Editor("kakoune", ROOT, GET_PLUGINS, generate_nix) - pluginupdate.update_plugins(editor) + editor = KakouneEditor("kakoune", ROOT, GET_PLUGINS) + parser = editor.create_parser() + args = parser.parse_args() + + pluginupdate.update_plugins(editor, args) if __name__ == "__main__": diff --git a/pkgs/development/lua-modules/generated-packages.nix b/pkgs/development/lua-modules/generated-packages.nix index 782084c19957..6bd6d7891e91 100644 --- a/pkgs/development/lua-modules/generated-packages.nix +++ b/pkgs/development/lua-modules/generated-packages.nix @@ -110,20 +110,20 @@ binaryheap = buildLuarocksPackage { bit32 = buildLuarocksPackage { pname = "bit32"; - version = "5.3.0-1"; + version = "5.3.5.1-1"; src = fetchurl { url = "https://luarocks.org/bit32-5.3.0-1.src.rock"; sha256 = "19i7kc2pfg9hc6qjq4kka43q6qk71bkl2rzvrjaks6283q6wfyzy"; }; - disabled = (luaOlder "5.1"); + disabled = (luaOlder "5.1") || (luaAtLeast "5.5"); propagatedBuildInputs = [ lua ]; meta = with lib; { homepage = "http://www.lua.org/manual/5.2/manual.html#6.7"; description = "Lua 5.2 bit manipulation library"; maintainers = with maintainers; [ lblasc ]; - license.fullName = "MIT/X11"; + license.fullName = "MIT"; }; }; @@ -169,16 +169,15 @@ cassowary = buildLuarocksPackage { license.fullName = "Apache 2"; }; }; - compat53 = buildLuarocksPackage { pname = "compat53"; - version = "0.7-1"; + version = "0.8-1"; src = fetchurl { url = "https://luarocks.org/compat53-0.7-1.src.rock"; sha256 = "0kpaxbpgrwjn4jjlb17fn29a09w6lw732d21bi0302kqcaixqpyb"; }; - disabled = (luaOlder "5.1") || (luaAtLeast "5.4"); + disabled = (luaOlder "5.1") || (luaAtLeast "5.5"); propagatedBuildInputs = [ lua ]; meta = with lib; { @@ -275,7 +274,7 @@ cyrussasl = buildLuarocksPackage { digestif = buildLuarocksPackage { pname = "digestif"; - version = "0.2-1"; + version = "0.4-1"; src = fetchurl { url = "mirror://luarocks/digestif-0.2-1.src.rock"; @@ -328,7 +327,7 @@ fifo = buildLuarocksPackage { http = buildLuarocksPackage { pname = "http"; - version = "0.3-0"; + version = "0.4-0"; src = fetchurl { url = "https://luarocks.org/http-0.3-0.src.rock"; @@ -1130,13 +1129,13 @@ luaevent = buildLuarocksPackage { luaexpat = buildLuarocksPackage { pname = "luaexpat"; - version = "1.3.0-1"; + version = "1.3.3-1"; src = fetchurl { url = "https://luarocks.org/luaexpat-1.3.0-1.src.rock"; sha256 = "15jqz5q12i9zvjyagzwz2lrpzya64mih8v1hxwr0wl2gsjh86y5a"; }; - disabled = (luaOlder "5.1"); + disabled = (luaOlder "5.0"); propagatedBuildInputs = [ lua ]; meta = with lib; { @@ -1167,7 +1166,7 @@ luaffi = buildLuarocksPackage { luafilesystem = buildLuarocksPackage { pname = "luafilesystem"; - version = "1.7.0-2"; + version = "1.8.0-1"; src = fetchurl { url = "https://luarocks.org/luafilesystem-1.7.0-2.src.rock"; @@ -1220,14 +1219,14 @@ luaossl = buildLuarocksPackage { luaposix = buildLuarocksPackage { pname = "luaposix"; - version = "34.1.1-1"; + version = "35.0-1"; src = fetchurl { url = "https://luarocks.org/luaposix-34.1.1-1.src.rock"; sha256 = "1l9pkn3g0nzlbmmfj12rhfwvkqb06c21ydqxqgmnmd3w9z4ck53w"; }; - disabled = (luaOlder "5.1") || (luaAtLeast "5.4"); - propagatedBuildInputs = [ bit32 lua ]; + disabled = (luaOlder "5.1") || (luaAtLeast "5.5"); + propagatedBuildInputs = [ lua ]; meta = with lib; { homepage = "http://github.com/luaposix/luaposix/"; @@ -1408,7 +1407,6 @@ luazip = buildLuarocksPackage { license.fullName = "MIT"; }; }; - luuid = buildLuarocksPackage { pname = "luuid"; version = "20120509-2"; @@ -1429,11 +1427,11 @@ luuid = buildLuarocksPackage { luv = buildLuarocksPackage { pname = "luv"; - version = "1.30.0-0"; + version = "1.41.1-0"; src = fetchurl { - url = "https://luarocks.org/luv-1.30.0-0.src.rock"; - sha256 = "1z5sdq9ld4sm5pws9qxpk9cadv9i7ycwad1zwsa57pj67gly11vi"; + url = "https://luarocks.org/luv-1.41.1-0.src.rock"; + sha256 = "0l1v07nhrkzsddbcc4bak382b5flyw6x8g4i394ylbfl25zwcmai"; }; disabled = (luaOlder "5.1"); propagatedBuildInputs = [ lua ]; @@ -1546,7 +1544,6 @@ mpack = buildLuarocksPackage { license.fullName = "MIT"; }; }; - nvim-client = buildLuarocksPackage { pname = "nvim-client"; version = "0.2.2-1"; diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index df948cc0e55d..fdb33df9729d 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -11,9 +11,14 @@ import inspect import os import sys +import logging +import textwrap from typing import List, Tuple from pathlib import Path +log = logging.getLogger() +log.addHandler(logging.StreamHandler()) + # Import plugin update library from maintainers/scripts/pluginupdate.py ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) sys.path.insert(0, os.path.join(ROOT.parent.parent.parent, "maintainers", "scripts")) @@ -40,50 +45,49 @@ HEADER = ( ) -def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str): - sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) +class VimEditor(pluginupdate.Editor): + def generate_nix(self, plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str): + sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) - with open(outfile, "w+") as f: - f.write(HEADER) - f.write( - """ -{ lib, buildVimPluginFrom2Nix, fetchFromGitHub }: + with open(outfile, "w+") as f: + f.write(HEADER) + f.write(textwrap.dedent(""" + { lib, buildVimPluginFrom2Nix, fetchFromGitHub }: -final: prev: -{""" - ) - for owner, repo, plugin in sorted_plugins: - if plugin.has_submodules: - submodule_attr = "\n fetchSubmodules = true;" - else: - submodule_attr = "" + final: prev: + {""" + )) + for owner, repo, plugin in sorted_plugins: + if plugin.has_submodules: + submodule_attr = "\n fetchSubmodules = true;" + else: + submodule_attr = "" + + f.write(textwrap.indent(textwrap.dedent( + f""" + {plugin.normalized_name} = buildVimPluginFrom2Nix {{ + pname = "{plugin.normalized_name}"; + version = "{plugin.version}"; + src = fetchFromGitHub {{ + owner = "{owner}"; + repo = "{repo}"; + rev = "{plugin.commit}"; + sha256 = "{plugin.sha256}";{submodule_attr} + }}; + meta.homepage = "https://github.com/{owner}/{repo}/"; + }}; + """ + ), ' ')) + f.write("\n}") + print(f"updated {outfile}") - f.write( - f""" - {plugin.normalized_name} = buildVimPluginFrom2Nix {{ - pname = "{plugin.normalized_name}"; - version = "{plugin.version}"; - src = fetchFromGitHub {{ - owner = "{owner}"; - repo = "{repo}"; - rev = "{plugin.commit}"; - sha256 = "{plugin.sha256}";{submodule_attr} - }}; - meta.homepage = "https://github.com/{owner}/{repo}/"; - }}; -""" - ) - f.write( - """ -} -""" - ) - print(f"updated {outfile}") def main(): - editor = pluginupdate.Editor("vim", ROOT, GET_PLUGINS, generate_nix) - pluginupdate.update_plugins(editor) + editor = VimEditor("vim", ROOT, GET_PLUGINS) + parser = editor.create_parser() + args = parser.parse_args() + pluginupdate.update_plugins(editor, args) if __name__ == "__main__":