From d255d416788a7b261252a55fa1c23b1fe3865f0f Mon Sep 17 00:00:00 2001 From: Thomas Strobel Date: Wed, 4 Feb 2015 22:00:18 +0100 Subject: [PATCH] Update: new features for nix-template-rpm - nix-template-rpm can now split the generated templates into a static part that goes into the nixpkgs tree a dynamic part that can be updated easily to track the rpm spec files - add lookup mechanism for package names and package paths - add mechanism to update existing nix-expression with new download files --- .../templaterpm/nix-template-rpm.py | 320 +++++++++++++++--- 1 file changed, 282 insertions(+), 38 deletions(-) diff --git a/pkgs/build-support/templaterpm/nix-template-rpm.py b/pkgs/build-support/templaterpm/nix-template-rpm.py index 42f8ee8a75fe..ec3a619027e7 100755 --- a/pkgs/build-support/templaterpm/nix-template-rpm.py +++ b/pkgs/build-support/templaterpm/nix-template-rpm.py @@ -4,6 +4,7 @@ import sys import os import subprocess import argparse +import re import shutil import rpm import urlparse @@ -14,8 +15,8 @@ import toposort -class NixTemplateRPM(object): - def __init__(self, specFilename, inputDir=None, maintainer="MAINTAINER"): +class SPECTemplate(object): + def __init__(self, specFilename, outputDir, inputDir=None, buildRootInclude=None, translateTable=None, repositoryDir=None, allPackagesDir=None, maintainer="MAINTAINER"): rpm.addMacro("buildroot","$out") rpm.addMacro("_libdir","lib") rpm.addMacro("_libexecdir","libexec") @@ -24,22 +25,41 @@ class NixTemplateRPM(object): rpm.addMacro("_topdir","SPACER_DIR_FOR_REMOVAL") rpm.addMacro("_sourcedir","SOURCE_DIR_SPACER") + self.packageGroups = [ "ocaml", "python" ] + ts = rpm.TransactionSet() self.specFilename = specFilename self.spec = ts.parseSpec(specFilename) self.inputDir = inputDir + self.buildRootInclude = buildRootInclude + self.repositoryDir = repositoryDir + self.allPackagesDir = allPackagesDir self.maintainer = maintainer - self.packageGroups = [ "ocaml", "python" ] + self.translateTable = translateTable + + self.facts = self.getFacts() + self.key = self.getSelfKey() + + tmpDir = os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name'])) + if self.translateTable != None: + self.relOutputDir = self.translateTable.path(self.key,tmpDir) + else: + self.relOutputDir = tmpDir + + self.final_output_dir = os.path.normpath( self.relOutputDir ) + + if self.repositoryDir != None: + self.potential_repository_dir = os.path.normpath( os.path.join(self.repositoryDir,self.relOutputDir) ) def rewriteCommands(self, string): string = string.replace('SPACER_DIR_FOR_REMOVAL/','') string = string.replace('SPACER_DIR_FOR_REMOVAL','') - string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/','${./')+'}' if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n'))) + string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/',('${./' if (self.buildRootInclude == None) else '${buildRoot}/usr/share/buildroot/SOURCES/'))+('}' if (self.buildRootInclude == None) else '') if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n'))) string = string.replace('\n','\n ') string = string.rstrip() return string @@ -85,18 +105,31 @@ class NixTemplateRPM(object): else: raise Exception("Unknown target") packages = [] + return packages def getBuildInputs(self,target=None): - return self.rewriteInputs(target,self.spec.sourceHeader['requires']) + inputs = self.rewriteInputs(target,self.spec.sourceHeader['requires']) + if self.translateTable != None: + return map(lambda x: self.translateTable.name(x), inputs) + else: + return inputs - def getSelf(self): + def getSelfKey(self): name = self.spec.sourceHeader['name'] if len(name.split('-')) > 1 and name.split('-')[0] in self.packageGroups: - return self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0] + key = self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0] else: - return self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0] + key = self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0] + return key + + def getSelf(self): + if self.translateTable != None: + return self.translateTable.name(self.key) + else: + return self.key + @@ -112,12 +145,34 @@ class NixTemplateRPM(object): shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename)) + def getFacts(self): + facts = {} + facts["name"] = self.rewriteName(self.spec.sourceHeader['name']) + facts["version"] = self.spec.sourceHeader['version'] + + facts["url"] = [] + facts["sha256"] = [] + sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ] + for url in sources: + p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, err = p.communicate() + sha256 = output[:-1] #remove new line + facts["url"].append(url) + facts["sha256"].append(sha256) + + patches = [source for (source, _, flag) in self.spec.sources if flag==2] + if self.buildRootInclude == None: + facts["patches"] = map(lambda x: './'+x, patches) + else: + facts["patches"] = map(lambda x: '"${buildRoot}/usr/share/buildroot/SOURCES/'+x+'"', reversed(patches)) + + return facts + @property def name(self): - out = 'stdenv.mkDerivation {\n' - out += ' name = "' + self.rewriteName(self.spec.sourceHeader['name']) + '-' + self.spec.sourceHeader['version'] + '";\n' - out += ' version = "' + self.spec.sourceHeader['version'] + '";\n' + out = ' name = "' + self.facts["name"] + '-' + self.facts["version"] + '";\n' + out += ' version = "' + self.facts['version'] + '";\n' return out @@ -125,10 +180,7 @@ class NixTemplateRPM(object): def src(self): sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ] out = '' - for url in sources: - p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, err = p.communicate() - sha256 = output[:-1] #remove new line + for (url,sha256) in zip(self.facts['url'],self.facts['sha256']): out += ' src = fetchurl {\n' out += ' url = "' + url + '";\n' out += ' sha256 = "' + sha256 + '";\n' @@ -138,8 +190,7 @@ class NixTemplateRPM(object): @property def patch(self): - patches = [source for (source, _, flag) in self.spec.sources if flag==2] - out = ' patches = [ ' + ' '.join(map(lambda x: './'+x, patches)) + ' ];\n' + out = ' patches = [ ' + ' '.join(self.facts['patches']) + ' ];\n' return out @@ -189,11 +240,29 @@ class NixTemplateRPM(object): return out - def __str__(self): - head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' - body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] - return head + '\n'.join(body) + head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' + head += 'stdenv.mkDerivation {\n' + body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] + return head + '\n'.join(body) + + + def getTemplate(self): + head = '{stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' + head += 'let\n' + head += ' buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n' + head += 'in\n\n' + head += 'stdenv.mkDerivation {\n' + head += ' inherit (buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+') name version src;\n' + head += ' patches = buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+'.patches ++ [];\n\n' + body = [ self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] + return head + '\n'.join(body) + + + def getInclude(self): + head = self.rewriteName(self.spec.sourceHeader['name']) + ' = {\n' + body = [ self.name, self.src, self.patch ] + return head + '\n'.join(body) + '};\n' def __cmp__(self,other): @@ -203,8 +272,8 @@ class NixTemplateRPM(object): return -1 - def callPackage(self, output_dir): - callPackage = ' ' + self.getSelf() + ' = callPackage ' + os.path.relpath(output_dir) + ' {' + def callPackage(self): + callPackage = ' ' + self.getSelf() + ' = callPackage ' + os.path.relpath(self.final_output_dir, self.allPackagesDir) + ' {' newline = False; for target in self.packageGroups: tmp = self.getBuildInputs(target) @@ -219,25 +288,141 @@ class NixTemplateRPM(object): - def generateTemplate(self, outputDir): - output_dir = os.path.normpath( os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name'])) ) - if not os.path.exists(output_dir): - os.makedirs(output_dir) + def generateCombined(self): + if not os.path.exists(self.final_output_dir): + os.makedirs(self.final_output_dir) if self.inputDir != None: - self.copySources(self.inputDir, output_dir) - self.copyPatches(self.inputDir, output_dir) + self.copySources(self.inputDir, self.final_output_dir) + self.copyPatches(self.inputDir, self.final_output_dir) - nixfile = open(os.path.join(output_dir,'default.nix'), 'w') + nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w') nixfile.write(str(self)) nixfile.close() - shutil.copyfile(self.specFilename, os.path.join(output_dir, os.path.basename(self.specFilename))) - - self.pkgCall = self.callPackage(output_dir) + shutil.copyfile(self.specFilename, os.path.join(self.final_output_dir, os.path.basename(self.specFilename))) + def generateSplit(self): + if not os.path.exists(self.final_output_dir): + os.makedirs(self.final_output_dir) + + nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w') + nixfile.write(self.getTemplate()) + nixfile.close() + + return self.getInclude() + + + + + + +class NixTemplate(object): + def __init__(self, nixfile): + self.nixfile = nixfile + self.original = { "name":None, "version":None, "url":None, "sha256":None, "patches":None } + self.update = { "name":None, "version":None, "url":None, "sha256":None, "patches":None } + self.matchedLines = {} + + if os.path.isfile(nixfile): + with file(nixfile, 'r') as infile: + for (n,line) in enumerate(infile): + name = re.match(r'^\s*name\s*=\s*"(.*?)"\s*;\s*$', line) + version = re.match(r'^\s*version\s*=\s*"(.*?)"\s*;\s*$', line) + url = re.match(r'^\s*url\s*=\s*"?(.*?)"?\s*;\s*$', line) + sha256 = re.match(r'^\s*sha256\s*=\s*"(.*?)"\s*;\s*$', line) + patches = re.match(r'^\s*patches\s*=\s*(\[.*?\])\s*;\s*$', line) + if name != None and self.original["name"] == None: + self.original["name"] = name.group(1) + self.matchedLines[n] = "name" + if version != None and self.original["version"] == None: + self.original["version"] = version.group(1) + self.matchedLines[n] = "version" + if url != None and self.original["url"] == None: + self.original["url"] = url.group(1) + self.matchedLines[n] = "url" + if sha256 != None and self.original["sha256"] == None: + self.original["sha256"] = sha256.group(1) + self.matchedLines[n] = "sha256" + if patches != None and self.original["patches"] == None: + self.original["patches"] = patches.group(1) + self.matchedLines[n] = "patches" + + + def generateUpdated(self, nixOut): + nixTemplateFile = open(os.path.normpath(self.nixfile),'r') + nixOutFile = open(os.path.normpath(nixOut),'w') + for (n,line) in enumerate(nixTemplateFile): + if self.matchedLines.has_key(n) and self.update[self.matchedLines[n]] != None: + nixOutFile.write(line.replace(self.original[self.matchedLines[n]], self.update[self.matchedLines[n]], 1)) + else: + nixOutFile.write(line) + nixTemplateFile.close() + nixOutFile.close() + + + def loadUpdate(self,orig): + if orig.has_key("name") and orig.has_key("version"): + self.update["name"] = orig["name"] + '-' + orig["version"] + self.update["version"] = orig["version"] + if orig.has_key("url") and orig.has_key("sha256") and len(orig["url"])>0: + self.update["url"] = orig["url"][0] + self.update["sha256"] = orig["sha256"][0] + for url in orig["url"][1:-1]: + sys.stderr.write("WARNING: URL has been dropped: %s\n" % url) + if orig.has_key("patches"): + self.update["patches"] = '[ ' + ' '.join(orig['patches']) + ' ]' + + +class TranslationTable(object): + def __init__(self): + self.tablePath = {} + self.tableName = {} + + def update(self, key, path, name=None): + self.tablePath[key] = path + if name != None: + self.tableName[key] = name + + def readTable(self, tableFile): + with file(tableFile, 'r') as infile: + for line in infile: + match = re.match(r'^(.+?)\s+(.+?)\s+(.+?)\s*$', line) + if match != None: + if not self.tablePath.has_key(match.group(1)): + self.tablePath[match.group(1)] = match.group(2) + if not self.tableName.has_key(match.group(1)): + self.tableName[match.group(1)] = match.group(3) + else: + match = re.match(r'^(.+?)\s+(.+?)\s*$', line) + if not self.tablePath.has_key(match.group(1)): + self.tablePath[match.group(1)] = match.group(2) + + def writeTable(self, tableFile): + outFile = open(os.path.normpath(tableFile),'w') + keys = self.tablePath.keys() + keys.sort() + for k in keys: + if self.tableName.has_key(k): + outFile.write( k + " " + self.tablePath[k] + " " + self.tableName[k] + "\n" ) + else: + outFile.write( k + " " + self.tablePath[k] + "\n" ) + outFile.close() + + def name(self, key): + if self.tableName.has_key(key): + return self.tableName[key] + else: + return key + + def path(self, key, orig): + if self.tablePath.has_key(key): + return self.tablePath[key] + else: + return orig + @@ -247,23 +432,65 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate .nix templates from RPM spec files") parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file") parser.add_argument("-o", "--output", metavar="OUT_DIR", required=True, help="output directory") - parser.add_argument("-i", "--input", metavar="IN_DIR", default=None, help="input directory") - parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", required=True, help="package maintainer") + parser.add_argument("-b", "--buildRoot", metavar="BUILDROOT_DIR", default=None, help="buildroot output directory") + parser.add_argument("-i", "--inputSources", metavar="IN_DIR", default=None, help="sources input directory") + parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", default="__NIX_MAINTAINER__", help="package maintainer") + parser.add_argument("-r", "--repository", metavar="REP_DIR", default=None, help="nix repository to compare output against") + parser.add_argument("-t", "--translate", metavar="TRANSLATE_TABLE", default=None, help="path of translation table for name and path") + parser.add_argument("-u", "--translateOut", metavar="TRANSLATE_OUT", default=None, help="output path for updated translation table") + parser.add_argument("-a", "--allPackages", metavar="ALL_PACKAGES", default=None, help="top level dir to call packages from") args = parser.parse_args() + allPackagesDir = os.path.normpath( os.path.dirname(args.allPackages) ) + if not os.path.exists(allPackagesDir): + os.makedirs(allPackagesDir) + buildRootContent = {} nameMap = {} + newTable = TranslationTable() + if args.translate != None: + table = TranslationTable() + table.readTable(args.translate) + newTable.readTable(args.translate) + else: + table = None + for specPath in args.specs: try: sys.stderr.write("INFO: generate nix file from: %s\n" % specPath) - spec = NixTemplateRPM(specPath, args.input, args.maintainer) - spec.generateTemplate(args.output) + + spec = SPECTemplate(specPath, args.output, args.inputSources, args.buildRoot, table, args.repository, allPackagesDir, args.maintainer) + if args.repository != None: + if os.path.exists(os.path.join(spec.potential_repository_dir,'default.nix')): + nixTemplate = NixTemplate(os.path.join(spec.potential_repository_dir,'default.nix')) + nixTemplate.loadUpdate(spec.facts) + if not os.path.exists(spec.final_output_dir): + os.makedirs(spec.final_output_dir) + nixTemplate.generateUpdated(os.path.join(spec.final_output_dir,'default.nix')) + else: + sys.stderr.write("WARNING: Repository does not contain template: %s\n" % os.path.join(spec.potential_repository_dir,'default.nix')) + if args.buildRoot == None: + spec.generateCombined() + else: + buildRootContent[spec.key] = spec.generateSplit() + else: + if args.buildRoot == None: + spec.generateCombined() + else: + buildRootContent[spec.key] = spec.generateSplit() + + newTable.update(spec.key,spec.relOutputDir,spec.getSelf()) nameMap[spec.getSelf()] = spec except Exception, e: sys.stderr.write("ERROR: %s failed with:\n%s\n%s\n" % (specPath,e.message,traceback.format_exc())) + if args.translateOut != None: + if not os.path.exists(os.path.dirname(os.path.normpath(args.translateOut))): + os.makedirs(os.path.dirname(os.path.normpath(args.translateOut))) + newTable.writeTable(args.translateOut) + graph = {} for k, v in nameMap.items(): graph[k] = set(v.getBuildInputs("ALL")) @@ -271,4 +498,21 @@ if __name__ == "__main__": sortedSpecs = toposort.toposort_flatten(graph) sortedSpecs = filter( lambda x: x in nameMap.keys(), sortedSpecs) - print '\n\n'.join(map(lambda x: x.pkgCall, map(lambda x: nameMap[x], sortedSpecs))) + allPackagesFile = open(os.path.normpath( args.allPackages ), 'w') + allPackagesFile.write( '\n\n'.join(map(lambda x: x.callPackage(), map(lambda x: nameMap[x], sortedSpecs))) ) + allPackagesFile.close() + + if args.buildRoot != None: + buildRootFilename = os.path.normpath( args.buildRoot ) + if not os.path.exists(os.path.dirname(buildRootFilename)): + os.makedirs(os.path.dirname(buildRootFilename)) + buildRootFile = open(buildRootFilename, 'w') + buildRootFile.write( "{ fetchurl, buildRoot }: {\n\n" ) + keys = buildRootContent.keys() + keys.sort() + for k in keys: + buildRootFile.write( buildRootContent[k] + '\n' ) + buildRootFile.write( "}\n" ) + buildRootFile.close() + +