ilyakooo0/nixpkgs
1
0
Fork 0
mirror of https://github.com/ilyakooo0/nixpkgs.git synced 2024-11-11 04:02:55 +03:00
Code Issues Projects Releases Wiki Activity

update-luarocks-packages: use pluginupdate.py

Browse Source 
Cleans up the common interface between the updaters.
Restores the ability to regen the lua packages in parallel.
This commit is contained in:
Matthieu Coudron 2021-07-30 02:25:15 +02:00
parent 4b88ab8c5c
commit 8ddad244ea
5 changed files with 236 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
maintainers/scripts/pluginupdate.py
View File

@ -42,8 +42,6 @@ LOG_LEVELS = {
}
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
"""Retry calling the decorated function using an exponential backoff.
@ -203,7 +201,6 @@ class Editor:
name: str,
root: Path,
get_plugins: str,
generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None],
default_in: Optional[Path] = None,
default_out: Optional[Path] = None,
deprecated: Optional[Path] = None,
@ -213,7 +210,6 @@ class Editor:
self.name = name
self.root = root
self.get_plugins = get_plugins
self._generate_nix = generate_nix
self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
self.default_out = default_out or root.joinpath("generated.nix")
self.deprecated = deprecated or root.joinpath("deprecated.json")
@ -226,9 +222,9 @@ class Editor:
def load_plugin_spec(self, plugin_file) -> List[PluginDesc]:
return load_plugin_spec(plugin_file)
def generate_nix(self, plugins, outfile):
def generate_nix(self, plugins, outfile: str):
'''Returns nothing for now, writes directly to outfile'''
self._generate_nix(plugins, outfile)
raise NotImplementedError()
def get_update(self, input_file: str, outfile: str, proc: int):
return get_update(input_file, outfile, proc, editor=self)
@ -237,9 +233,58 @@ class Editor:
def attr_path(self):
return self.name + "Plugins"
def get_drv_name(self, name: str):
return self.attr_path + "." + name
def rewrite_input(self, *args, **kwargs):
return rewrite_input(*args, **kwargs)
def create_parser(self):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {self.name} plugins"
f"By default from {self.default_in} to {self.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=self.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=self.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser
@ -466,54 +511,6 @@ def rewrite_input(
with open(input_file, "w") as f:
f.writelines(lines)
# TODO move to Editor ?
def parse_args(editor: Editor):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {editor.name} plugins"
f"By default from {editor.default_in} to {editor.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {editor.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=editor.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=editor.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser.parse_args()
def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
repo.index.add([str(f.resolve()) for f in files])
@ -547,12 +544,10 @@ def get_update(input_file: str, outfile: str, proc: int, editor: Editor):
return update
def update_plugins(editor: Editor):
def update_plugins(editor: Editor, args):
"""The main entry function of this module. All input arguments are grouped in the `Editor`."""
args = parse_args(editor)
log.setLevel(LOG_LEVELS[args.debug])
log.info("Start updating plugins")
nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
update = editor.get_update(args.input_file, args.outfile, args.proc)
@ -581,7 +576,7 @@ def update_plugins(editor: Editor):
if autocommit:
commit(
nixpkgs_repo,
"{editor.attr_path}.{name}: init at {version}".format(
"{editor.get_drv_name name}: init at {version}".format(
editor=editor.name, name=plugin.normalized_name, version=plugin.version
),
[args.outfile, args.input_file],

121
maintainers/scripts/update-luarocks-packages
View File

@ -16,20 +16,17 @@ from dataclasses import dataclass
import subprocess
import csv
import logging
import textwrap
from multiprocessing.dummy import Pool
from typing import List
from typing import List, Tuple
from pathlib import Path
LOG_LEVELS = {
logging.getLevelName(level): level for level in [
logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
}
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent
from pluginupdate import Editor, parse_args, update_plugins, PluginDesc, CleanEnvironment
from pluginupdate import Editor, update_plugins, PluginDesc, CleanEnvironment, LOG_LEVELS, Cache
PKG_LIST="maintainers/scripts/luarocks-packages.csv"
TMP_FILE="$(mktemp)"
@ -67,12 +64,11 @@ class LuaEditor(Editor):
def get_current_plugins(self):
return []
def load_plugin_spec(self, input_file) -> List[PluginDesc]:
def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
luaPackages = []
csvfilename=input_file
log.info("Loading package descriptions from %s", csvfilename)
with open(csvfilename, newline='') as csvfile:
reader = csv.DictReader(csvfile,)
for row in reader:
@ -81,36 +77,81 @@ class LuaEditor(Editor):
luaPackages.append(plugin)
return luaPackages
def generate_nix(
self,
results: List[Tuple[LuaPlugin, str]],
outfilename: str
):
with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
header2 = textwrap.dedent(
# header2 = inspect.cleandoc(
"""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
f.write(header2)
for (plugin, nix_expr) in results:
f.write(f"{plugin.normalized_name} = {nix_expr}")
f.write(FOOTER)
f.flush()
# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)
print(f"updated {outfilename}")
@property
def attr_path(self):
return "luaPackages"
def get_update(self, input_file: str, outfile: str, _: int):
def get_update(self, input_file: str, outfile: str, proc: int):
cache: Cache = Cache(self.cache_file)
_prefetch = generate_pkg_nix
def update() -> dict:
plugin_specs = self.load_plugin_spec(input_file)
sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
self.generate_nix(plugin_specs, outfile)
try:
pool = Pool(processes=proc)
results = pool.map(_prefetch, sorted_plugin_specs)
finally:
pass
self.generate_nix(results, outfile)
redirects = []
return redirects
return update
def rewrite_input(self, *args, **kwargs):
def rewrite_input(self, input_file: str, *args, **kwargs):
# vim plugin reads the file before update but that shouldn't be our case
# not implemented yet
# fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
# input_file = "toto.csv"
# with open(input_file, newline='') as csvfile:
# writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# writer.writeheader()
# for row in reader:
# # name,server,version,luaversion,maintainers
# plugin = LuaPlugin(**row)
# luaPackages.append(plugin)
pass
def generate_nix(
plugins: List[LuaPlugin],
outfilename: str
):
sorted_plugins = sorted(plugins, key=lambda v: v.name.lower())
# plug = {}
# selon le manifest luarocks.org/manifest
def _generate_pkg_nix(plug):
def generate_pkg_nix(plug: LuaPlugin):
'''
Generate nix expression for a luarocks package
Our cache key associates "p.name-p.version" to its rockspec
'''
log.debug("Generating nix expression for %s", plug.name)
cmd = [ "luarocks", "nix", plug.name]
if plug.server:
cmd.append(f"--only-server={plug.server}")
@ -125,52 +166,26 @@ def generate_nix(
local_pkgs = str(ROOT.resolve())
cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]
log.debug("running %s", cmd2)
log.debug("running %s", ' '.join(cmd2))
lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
cmd.append(f"--lua-dir={lua_drv_path}/bin")
log.debug("running %s", cmd)
output = subprocess.check_output(cmd, text=True)
return output
with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
f.write("""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
for plugin in sorted_plugins:
nix_expr = _generate_pkg_nix(plugin)
f.write(f"{plugin.normalized_name} = {nix_expr}"
)
f.write(FOOTER)
f.flush()
# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)
print(f"updated {outfilename}")
def load_plugin_spec():
pass
return (plug, output)
def main():
editor = LuaEditor("lua", ROOT, '', generate_nix,
editor = LuaEditor("lua", ROOT, '',
default_in = ROOT.joinpath(PKG_LIST),
default_out = ROOT.joinpath(GENERATED_NIXFILE)
)
args = parse_args(editor)
parser = editor.create_parser()
args = parser.parse_args()
log.setLevel(LOG_LEVELS[args.debug])
update_plugins(editor)
update_plugins(editor, args)
if __name__ == "__main__":

25
pkgs/applications/editors/kakoune/plugins/update.py
View File

@ -39,18 +39,20 @@ in lib.filterAttrs (n: v: v != null) checksums)"""
HEADER = "# This file has been generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"
class KakouneEditor(pluginupdate.Editor):
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
"""
{ lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
let
{ lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
let
packages = ( self:
{"""
{"""
)
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
@ -71,20 +73,23 @@ let
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
"""
)
f.write(
"""
});
in lib.fix' (lib.extends overrides packages)
"""
});
in lib.fix' (lib.extends overrides packages)
"""
)
print(f"updated {outfile}")
def main():
editor = pluginupdate.Editor("kakoune", ROOT, GET_PLUGINS, generate_nix)
pluginupdate.update_plugins(editor)
editor = KakouneEditor("kakoune", ROOT, GET_PLUGINS)
parser = editor.create_parser()
args = parser.parse_args()
pluginupdate.update_plugins(editor, args)
if __name__ == "__main__":

35
pkgs/development/lua-modules/generated-packages.nix
View File

@ -110,20 +110,20 @@ binaryheap = buildLuarocksPackage {
bit32 = buildLuarocksPackage {
pname = "bit32";
version = "5.3.0-1";
version = "5.3.5.1-1";
src = fetchurl {
url = "https://luarocks.org/bit32-5.3.0-1.src.rock";
sha256 = "19i7kc2pfg9hc6qjq4kka43q6qk71bkl2rzvrjaks6283q6wfyzy";
};
disabled = (luaOlder "5.1");
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
homepage = "http://www.lua.org/manual/5.2/manual.html#6.7";
description = "Lua 5.2 bit manipulation library";
maintainers = with maintainers; [ lblasc ];
license.fullName = "MIT/X11";
license.fullName = "MIT";
};
};
@ -169,16 +169,15 @@ cassowary = buildLuarocksPackage {
license.fullName = "Apache 2";
};
};
compat53 = buildLuarocksPackage {
pname = "compat53";
version = "0.7-1";
version = "0.8-1";
src = fetchurl {
url = "https://luarocks.org/compat53-0.7-1.src.rock";
sha256 = "0kpaxbpgrwjn4jjlb17fn29a09w6lw732d21bi0302kqcaixqpyb";
};
disabled = (luaOlder "5.1") || (luaAtLeast "5.4");
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
@ -275,7 +274,7 @@ cyrussasl = buildLuarocksPackage {
digestif = buildLuarocksPackage {
pname = "digestif";
version = "0.2-1";
version = "0.4-1";
src = fetchurl {
url = "mirror://luarocks/digestif-0.2-1.src.rock";
@ -328,7 +327,7 @@ fifo = buildLuarocksPackage {
http = buildLuarocksPackage {
pname = "http";
version = "0.3-0";
version = "0.4-0";
src = fetchurl {
url = "https://luarocks.org/http-0.3-0.src.rock";
@ -1130,13 +1129,13 @@ luaevent = buildLuarocksPackage {
luaexpat = buildLuarocksPackage {
pname = "luaexpat";
version = "1.3.0-1";
version = "1.3.3-1";
src = fetchurl {
url = "https://luarocks.org/luaexpat-1.3.0-1.src.rock";
sha256 = "15jqz5q12i9zvjyagzwz2lrpzya64mih8v1hxwr0wl2gsjh86y5a";
};
disabled = (luaOlder "5.1");
disabled = (luaOlder "5.0");
propagatedBuildInputs = [ lua ];
meta = with lib; {
@ -1167,7 +1166,7 @@ luaffi = buildLuarocksPackage {
luafilesystem = buildLuarocksPackage {
pname = "luafilesystem";
version = "1.7.0-2";
version = "1.8.0-1";
src = fetchurl {
url = "https://luarocks.org/luafilesystem-1.7.0-2.src.rock";
@ -1220,14 +1219,14 @@ luaossl = buildLuarocksPackage {
luaposix = buildLuarocksPackage {
pname = "luaposix";
version = "34.1.1-1";
version = "35.0-1";
src = fetchurl {
url = "https://luarocks.org/luaposix-34.1.1-1.src.rock";
sha256 = "1l9pkn3g0nzlbmmfj12rhfwvkqb06c21ydqxqgmnmd3w9z4ck53w";
};
disabled = (luaOlder "5.1") || (luaAtLeast "5.4");
propagatedBuildInputs = [ bit32 lua ];
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
homepage = "http://github.com/luaposix/luaposix/";
@ -1408,7 +1407,6 @@ luazip = buildLuarocksPackage {
license.fullName = "MIT";
};
};
luuid = buildLuarocksPackage {
pname = "luuid";
version = "20120509-2";
@ -1429,11 +1427,11 @@ luuid = buildLuarocksPackage {
luv = buildLuarocksPackage {
pname = "luv";
version = "1.30.0-0";
version = "1.41.1-0";
src = fetchurl {
url = "https://luarocks.org/luv-1.30.0-0.src.rock";
sha256 = "1z5sdq9ld4sm5pws9qxpk9cadv9i7ycwad1zwsa57pj67gly11vi";
url = "https://luarocks.org/luv-1.41.1-0.src.rock";
sha256 = "0l1v07nhrkzsddbcc4bak382b5flyw6x8g4i394ylbfl25zwcmai";
};
disabled = (luaOlder "5.1");
propagatedBuildInputs = [ lua ];
@ -1546,7 +1544,6 @@ mpack = buildLuarocksPackage {
license.fullName = "MIT";
};
};
nvim-client = buildLuarocksPackage {
pname = "nvim-client";
version = "0.2.2-1";

36
pkgs/misc/vim-plugins/update.py
View File

@ -11,9 +11,14 @@
import inspect
import os
import sys
import logging
import textwrap
from typing import List, Tuple
from pathlib import Path
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
# Import plugin update library from maintainers/scripts/pluginupdate.py
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))
sys.path.insert(0, os.path.join(ROOT.parent.parent.parent, "maintainers", "scripts"))
@ -40,25 +45,25 @@ HEADER = (
)
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
class VimEditor(pluginupdate.Editor):
def generate_nix(self, plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
"""
{ lib, buildVimPluginFrom2Nix, fetchFromGitHub }:
f.write(textwrap.dedent("""
{ lib, buildVimPluginFrom2Nix, fetchFromGitHub }:
final: prev:
{"""
)
final: prev:
{"""
))
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
f.write(
f.write(textwrap.indent(textwrap.dedent(
f"""
{plugin.normalized_name} = buildVimPluginFrom2Nix {{
pname = "{plugin.normalized_name}";
@ -71,19 +76,18 @@ final: prev:
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
)
f.write(
"""
}
"""
)
), ' '))
f.write("\n}")
print(f"updated {outfile}")
def main():
editor = pluginupdate.Editor("vim", ROOT, GET_PLUGINS, generate_nix)
pluginupdate.update_plugins(editor)
editor = VimEditor("vim", ROOT, GET_PLUGINS)
parser = editor.create_parser()
args = parser.parse_args()
pluginupdate.update_plugins(editor, args)
if __name__ == "__main__":