mirror of
https://github.com/ilyakooo0/nixpkgs.git
synced 2024-11-10 16:45:51 +03:00
Update: new features for nix-template-rpm
- nix-template-rpm can now split the generated templates into
a static part that goes into the nixpkgs tree
a dynamic part that can be updated easily to track the rpm spec files
- add lookup mechanism for package names and package paths
- add mechanism to update existing nix-expression with new download files
This commit is contained in:
Thomas Strobel
Rok Garbas
parent
e4c0ee53d2
commit
d255d41678
@ -4,6 +4,7 @@ import sys
|
||||
import os
|
||||
import subprocess
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import rpm
|
||||
import urlparse
|
||||
@ -14,8 +15,8 @@ import toposort
|
||||
|
||||
|
||||
|
||||
class NixTemplateRPM(object):
|
||||
def __init__(self, specFilename, inputDir=None, maintainer="MAINTAINER"):
|
||||
class SPECTemplate(object):
|
||||
def __init__(self, specFilename, outputDir, inputDir=None, buildRootInclude=None, translateTable=None, repositoryDir=None, allPackagesDir=None, maintainer="MAINTAINER"):
|
||||
rpm.addMacro("buildroot","$out")
|
||||
rpm.addMacro("_libdir","lib")
|
||||
rpm.addMacro("_libexecdir","libexec")
|
||||
@ -24,22 +25,41 @@ class NixTemplateRPM(object):
|
||||
rpm.addMacro("_topdir","SPACER_DIR_FOR_REMOVAL")
|
||||
rpm.addMacro("_sourcedir","SOURCE_DIR_SPACER")
|
||||
|
||||
self.packageGroups = [ "ocaml", "python" ]
|
||||
|
||||
ts = rpm.TransactionSet()
|
||||
|
||||
self.specFilename = specFilename
|
||||
self.spec = ts.parseSpec(specFilename)
|
||||
|
||||
self.inputDir = inputDir
|
||||
self.buildRootInclude = buildRootInclude
|
||||
self.repositoryDir = repositoryDir
|
||||
self.allPackagesDir = allPackagesDir
|
||||
self.maintainer = maintainer
|
||||
|
||||
self.packageGroups = [ "ocaml", "python" ]
|
||||
self.translateTable = translateTable
|
||||
|
||||
self.facts = self.getFacts()
|
||||
self.key = self.getSelfKey()
|
||||
|
||||
tmpDir = os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name']))
|
||||
if self.translateTable != None:
|
||||
self.relOutputDir = self.translateTable.path(self.key,tmpDir)
|
||||
else:
|
||||
self.relOutputDir = tmpDir
|
||||
|
||||
self.final_output_dir = os.path.normpath( self.relOutputDir )
|
||||
|
||||
if self.repositoryDir != None:
|
||||
self.potential_repository_dir = os.path.normpath( os.path.join(self.repositoryDir,self.relOutputDir) )
|
||||
|
||||
|
||||
|
||||
def rewriteCommands(self, string):
|
||||
string = string.replace('SPACER_DIR_FOR_REMOVAL/','')
|
||||
string = string.replace('SPACER_DIR_FOR_REMOVAL','')
|
||||
string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/','${./')+'}' if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n')))
|
||||
string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/',('${./' if (self.buildRootInclude == None) else '${buildRoot}/usr/share/buildroot/SOURCES/'))+('}' if (self.buildRootInclude == None) else '') if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n')))
|
||||
string = string.replace('\n','\n ')
|
||||
string = string.rstrip()
|
||||
return string
|
||||
@ -85,18 +105,31 @@ class NixTemplateRPM(object):
|
||||
else:
|
||||
raise Exception("Unknown target")
|
||||
packages = []
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def getBuildInputs(self,target=None):
|
||||
return self.rewriteInputs(target,self.spec.sourceHeader['requires'])
|
||||
inputs = self.rewriteInputs(target,self.spec.sourceHeader['requires'])
|
||||
if self.translateTable != None:
|
||||
return map(lambda x: self.translateTable.name(x), inputs)
|
||||
else:
|
||||
return inputs
|
||||
|
||||
def getSelf(self):
|
||||
def getSelfKey(self):
|
||||
name = self.spec.sourceHeader['name']
|
||||
if len(name.split('-')) > 1 and name.split('-')[0] in self.packageGroups:
|
||||
return self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0]
|
||||
key = self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0]
|
||||
else:
|
||||
return self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0]
|
||||
key = self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0]
|
||||
return key
|
||||
|
||||
def getSelf(self):
|
||||
if self.translateTable != None:
|
||||
return self.translateTable.name(self.key)
|
||||
else:
|
||||
return self.key
|
||||
|
||||
|
||||
|
||||
|
||||
@ -112,12 +145,34 @@ class NixTemplateRPM(object):
|
||||
shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename))
|
||||
|
||||
|
||||
def getFacts(self):
|
||||
facts = {}
|
||||
facts["name"] = self.rewriteName(self.spec.sourceHeader['name'])
|
||||
facts["version"] = self.spec.sourceHeader['version']
|
||||
|
||||
facts["url"] = []
|
||||
facts["sha256"] = []
|
||||
sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
|
||||
for url in sources:
|
||||
p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output, err = p.communicate()
|
||||
sha256 = output[:-1] #remove new line
|
||||
facts["url"].append(url)
|
||||
facts["sha256"].append(sha256)
|
||||
|
||||
patches = [source for (source, _, flag) in self.spec.sources if flag==2]
|
||||
if self.buildRootInclude == None:
|
||||
facts["patches"] = map(lambda x: './'+x, patches)
|
||||
else:
|
||||
facts["patches"] = map(lambda x: '"${buildRoot}/usr/share/buildroot/SOURCES/'+x+'"', reversed(patches))
|
||||
|
||||
return facts
|
||||
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
out = 'stdenv.mkDerivation {\n'
|
||||
out += ' name = "' + self.rewriteName(self.spec.sourceHeader['name']) + '-' + self.spec.sourceHeader['version'] + '";\n'
|
||||
out += ' version = "' + self.spec.sourceHeader['version'] + '";\n'
|
||||
out = ' name = "' + self.facts["name"] + '-' + self.facts["version"] + '";\n'
|
||||
out += ' version = "' + self.facts['version'] + '";\n'
|
||||
return out
|
||||
|
||||
|
||||
@ -125,10 +180,7 @@ class NixTemplateRPM(object):
|
||||
def src(self):
|
||||
sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
|
||||
out = ''
|
||||
for url in sources:
|
||||
p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output, err = p.communicate()
|
||||
sha256 = output[:-1] #remove new line
|
||||
for (url,sha256) in zip(self.facts['url'],self.facts['sha256']):
|
||||
out += ' src = fetchurl {\n'
|
||||
out += ' url = "' + url + '";\n'
|
||||
out += ' sha256 = "' + sha256 + '";\n'
|
||||
@ -138,8 +190,7 @@ class NixTemplateRPM(object):
|
||||
|
||||
@property
|
||||
def patch(self):
|
||||
patches = [source for (source, _, flag) in self.spec.sources if flag==2]
|
||||
out = ' patches = [ ' + ' '.join(map(lambda x: './'+x, patches)) + ' ];\n'
|
||||
out = ' patches = [ ' + ' '.join(self.facts['patches']) + ' ];\n'
|
||||
return out
|
||||
|
||||
|
||||
@ -189,11 +240,29 @@ class NixTemplateRPM(object):
|
||||
return out
|
||||
|
||||
|
||||
|
||||
def __str__(self):
|
||||
head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
|
||||
body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
|
||||
return head + '\n'.join(body)
|
||||
head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
|
||||
head += 'stdenv.mkDerivation {\n'
|
||||
body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
|
||||
return head + '\n'.join(body)
|
||||
|
||||
|
||||
def getTemplate(self):
|
||||
head = '{stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
|
||||
head += 'let\n'
|
||||
head += ' buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n'
|
||||
head += 'in\n\n'
|
||||
head += 'stdenv.mkDerivation {\n'
|
||||
head += ' inherit (buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+') name version src;\n'
|
||||
head += ' patches = buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+'.patches ++ [];\n\n'
|
||||
body = [ self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
|
||||
return head + '\n'.join(body)
|
||||
|
||||
|
||||
def getInclude(self):
|
||||
head = self.rewriteName(self.spec.sourceHeader['name']) + ' = {\n'
|
||||
body = [ self.name, self.src, self.patch ]
|
||||
return head + '\n'.join(body) + '};\n'
|
||||
|
||||
|
||||
def __cmp__(self,other):
|
||||
@ -203,8 +272,8 @@ class NixTemplateRPM(object):
|
||||
return -1
|
||||
|
||||
|
||||
def callPackage(self, output_dir):
|
||||
callPackage = ' ' + self.getSelf() + ' = callPackage ' + os.path.relpath(output_dir) + ' {'
|
||||
def callPackage(self):
|
||||
callPackage = ' ' + self.getSelf() + ' = callPackage ' + os.path.relpath(self.final_output_dir, self.allPackagesDir) + ' {'
|
||||
newline = False;
|
||||
for target in self.packageGroups:
|
||||
tmp = self.getBuildInputs(target)
|
||||
@ -219,25 +288,141 @@ class NixTemplateRPM(object):
|
||||
|
||||
|
||||
|
||||
def generateTemplate(self, outputDir):
|
||||
output_dir = os.path.normpath( os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name'])) )
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
def generateCombined(self):
|
||||
if not os.path.exists(self.final_output_dir):
|
||||
os.makedirs(self.final_output_dir)
|
||||
|
||||
if self.inputDir != None:
|
||||
self.copySources(self.inputDir, output_dir)
|
||||
self.copyPatches(self.inputDir, output_dir)
|
||||
self.copySources(self.inputDir, self.final_output_dir)
|
||||
self.copyPatches(self.inputDir, self.final_output_dir)
|
||||
|
||||
nixfile = open(os.path.join(output_dir,'default.nix'), 'w')
|
||||
nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
|
||||
nixfile.write(str(self))
|
||||
nixfile.close()
|
||||
|
||||
shutil.copyfile(self.specFilename, os.path.join(output_dir, os.path.basename(self.specFilename)))
|
||||
|
||||
self.pkgCall = self.callPackage(output_dir)
|
||||
shutil.copyfile(self.specFilename, os.path.join(self.final_output_dir, os.path.basename(self.specFilename)))
|
||||
|
||||
|
||||
|
||||
def generateSplit(self):
|
||||
if not os.path.exists(self.final_output_dir):
|
||||
os.makedirs(self.final_output_dir)
|
||||
|
||||
nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
|
||||
nixfile.write(self.getTemplate())
|
||||
nixfile.close()
|
||||
|
||||
return self.getInclude()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class NixTemplate(object):
|
||||
def __init__(self, nixfile):
|
||||
self.nixfile = nixfile
|
||||
self.original = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
|
||||
self.update = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
|
||||
self.matchedLines = {}
|
||||
|
||||
if os.path.isfile(nixfile):
|
||||
with file(nixfile, 'r') as infile:
|
||||
for (n,line) in enumerate(infile):
|
||||
name = re.match(r'^\s*name\s*=\s*"(.*?)"\s*;\s*$', line)
|
||||
version = re.match(r'^\s*version\s*=\s*"(.*?)"\s*;\s*$', line)
|
||||
url = re.match(r'^\s*url\s*=\s*"?(.*?)"?\s*;\s*$', line)
|
||||
sha256 = re.match(r'^\s*sha256\s*=\s*"(.*?)"\s*;\s*$', line)
|
||||
patches = re.match(r'^\s*patches\s*=\s*(\[.*?\])\s*;\s*$', line)
|
||||
if name != None and self.original["name"] == None:
|
||||
self.original["name"] = name.group(1)
|
||||
self.matchedLines[n] = "name"
|
||||
if version != None and self.original["version"] == None:
|
||||
self.original["version"] = version.group(1)
|
||||
self.matchedLines[n] = "version"
|
||||
if url != None and self.original["url"] == None:
|
||||
self.original["url"] = url.group(1)
|
||||
self.matchedLines[n] = "url"
|
||||
if sha256 != None and self.original["sha256"] == None:
|
||||
self.original["sha256"] = sha256.group(1)
|
||||
self.matchedLines[n] = "sha256"
|
||||
if patches != None and self.original["patches"] == None:
|
||||
self.original["patches"] = patches.group(1)
|
||||
self.matchedLines[n] = "patches"
|
||||
|
||||
|
||||
def generateUpdated(self, nixOut):
|
||||
nixTemplateFile = open(os.path.normpath(self.nixfile),'r')
|
||||
nixOutFile = open(os.path.normpath(nixOut),'w')
|
||||
for (n,line) in enumerate(nixTemplateFile):
|
||||
if self.matchedLines.has_key(n) and self.update[self.matchedLines[n]] != None:
|
||||
nixOutFile.write(line.replace(self.original[self.matchedLines[n]], self.update[self.matchedLines[n]], 1))
|
||||
else:
|
||||
nixOutFile.write(line)
|
||||
nixTemplateFile.close()
|
||||
nixOutFile.close()
|
||||
|
||||
|
||||
def loadUpdate(self,orig):
|
||||
if orig.has_key("name") and orig.has_key("version"):
|
||||
self.update["name"] = orig["name"] + '-' + orig["version"]
|
||||
self.update["version"] = orig["version"]
|
||||
if orig.has_key("url") and orig.has_key("sha256") and len(orig["url"])>0:
|
||||
self.update["url"] = orig["url"][0]
|
||||
self.update["sha256"] = orig["sha256"][0]
|
||||
for url in orig["url"][1:-1]:
|
||||
sys.stderr.write("WARNING: URL has been dropped: %s\n" % url)
|
||||
if orig.has_key("patches"):
|
||||
self.update["patches"] = '[ ' + ' '.join(orig['patches']) + ' ]'
|
||||
|
||||
|
||||
class TranslationTable(object):
|
||||
def __init__(self):
|
||||
self.tablePath = {}
|
||||
self.tableName = {}
|
||||
|
||||
def update(self, key, path, name=None):
|
||||
self.tablePath[key] = path
|
||||
if name != None:
|
||||
self.tableName[key] = name
|
||||
|
||||
def readTable(self, tableFile):
|
||||
with file(tableFile, 'r') as infile:
|
||||
for line in infile:
|
||||
match = re.match(r'^(.+?)\s+(.+?)\s+(.+?)\s*$', line)
|
||||
if match != None:
|
||||
if not self.tablePath.has_key(match.group(1)):
|
||||
self.tablePath[match.group(1)] = match.group(2)
|
||||
if not self.tableName.has_key(match.group(1)):
|
||||
self.tableName[match.group(1)] = match.group(3)
|
||||
else:
|
||||
match = re.match(r'^(.+?)\s+(.+?)\s*$', line)
|
||||
if not self.tablePath.has_key(match.group(1)):
|
||||
self.tablePath[match.group(1)] = match.group(2)
|
||||
|
||||
def writeTable(self, tableFile):
|
||||
outFile = open(os.path.normpath(tableFile),'w')
|
||||
keys = self.tablePath.keys()
|
||||
keys.sort()
|
||||
for k in keys:
|
||||
if self.tableName.has_key(k):
|
||||
outFile.write( k + " " + self.tablePath[k] + " " + self.tableName[k] + "\n" )
|
||||
else:
|
||||
outFile.write( k + " " + self.tablePath[k] + "\n" )
|
||||
outFile.close()
|
||||
|
||||
def name(self, key):
|
||||
if self.tableName.has_key(key):
|
||||
return self.tableName[key]
|
||||
else:
|
||||
return key
|
||||
|
||||
def path(self, key, orig):
|
||||
if self.tablePath.has_key(key):
|
||||
return self.tablePath[key]
|
||||
else:
|
||||
return orig
|
||||
|
||||
|
||||
|
||||
|
||||
@ -247,23 +432,65 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Generate .nix templates from RPM spec files")
|
||||
parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file")
|
||||
parser.add_argument("-o", "--output", metavar="OUT_DIR", required=True, help="output directory")
|
||||
parser.add_argument("-i", "--input", metavar="IN_DIR", default=None, help="input directory")
|
||||
parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", required=True, help="package maintainer")
|
||||
parser.add_argument("-b", "--buildRoot", metavar="BUILDROOT_DIR", default=None, help="buildroot output directory")
|
||||
parser.add_argument("-i", "--inputSources", metavar="IN_DIR", default=None, help="sources input directory")
|
||||
parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", default="__NIX_MAINTAINER__", help="package maintainer")
|
||||
parser.add_argument("-r", "--repository", metavar="REP_DIR", default=None, help="nix repository to compare output against")
|
||||
parser.add_argument("-t", "--translate", metavar="TRANSLATE_TABLE", default=None, help="path of translation table for name and path")
|
||||
parser.add_argument("-u", "--translateOut", metavar="TRANSLATE_OUT", default=None, help="output path for updated translation table")
|
||||
parser.add_argument("-a", "--allPackages", metavar="ALL_PACKAGES", default=None, help="top level dir to call packages from")
|
||||
args = parser.parse_args()
|
||||
|
||||
allPackagesDir = os.path.normpath( os.path.dirname(args.allPackages) )
|
||||
if not os.path.exists(allPackagesDir):
|
||||
os.makedirs(allPackagesDir)
|
||||
|
||||
buildRootContent = {}
|
||||
nameMap = {}
|
||||
|
||||
newTable = TranslationTable()
|
||||
if args.translate != None:
|
||||
table = TranslationTable()
|
||||
table.readTable(args.translate)
|
||||
newTable.readTable(args.translate)
|
||||
else:
|
||||
table = None
|
||||
|
||||
for specPath in args.specs:
|
||||
try:
|
||||
sys.stderr.write("INFO: generate nix file from: %s\n" % specPath)
|
||||
spec = NixTemplateRPM(specPath, args.input, args.maintainer)
|
||||
spec.generateTemplate(args.output)
|
||||
|
||||
spec = SPECTemplate(specPath, args.output, args.inputSources, args.buildRoot, table, args.repository, allPackagesDir, args.maintainer)
|
||||
if args.repository != None:
|
||||
if os.path.exists(os.path.join(spec.potential_repository_dir,'default.nix')):
|
||||
nixTemplate = NixTemplate(os.path.join(spec.potential_repository_dir,'default.nix'))
|
||||
nixTemplate.loadUpdate(spec.facts)
|
||||
if not os.path.exists(spec.final_output_dir):
|
||||
os.makedirs(spec.final_output_dir)
|
||||
nixTemplate.generateUpdated(os.path.join(spec.final_output_dir,'default.nix'))
|
||||
else:
|
||||
sys.stderr.write("WARNING: Repository does not contain template: %s\n" % os.path.join(spec.potential_repository_dir,'default.nix'))
|
||||
if args.buildRoot == None:
|
||||
spec.generateCombined()
|
||||
else:
|
||||
buildRootContent[spec.key] = spec.generateSplit()
|
||||
else:
|
||||
if args.buildRoot == None:
|
||||
spec.generateCombined()
|
||||
else:
|
||||
buildRootContent[spec.key] = spec.generateSplit()
|
||||
|
||||
newTable.update(spec.key,spec.relOutputDir,spec.getSelf())
|
||||
nameMap[spec.getSelf()] = spec
|
||||
|
||||
except Exception, e:
|
||||
sys.stderr.write("ERROR: %s failed with:\n%s\n%s\n" % (specPath,e.message,traceback.format_exc()))
|
||||
|
||||
if args.translateOut != None:
|
||||
if not os.path.exists(os.path.dirname(os.path.normpath(args.translateOut))):
|
||||
os.makedirs(os.path.dirname(os.path.normpath(args.translateOut)))
|
||||
newTable.writeTable(args.translateOut)
|
||||
|
||||
graph = {}
|
||||
for k, v in nameMap.items():
|
||||
graph[k] = set(v.getBuildInputs("ALL"))
|
||||
@ -271,4 +498,21 @@ if __name__ == "__main__":
|
||||
sortedSpecs = toposort.toposort_flatten(graph)
|
||||
sortedSpecs = filter( lambda x: x in nameMap.keys(), sortedSpecs)
|
||||
|
||||
print '\n\n'.join(map(lambda x: x.pkgCall, map(lambda x: nameMap[x], sortedSpecs)))
|
||||
allPackagesFile = open(os.path.normpath( args.allPackages ), 'w')
|
||||
allPackagesFile.write( '\n\n'.join(map(lambda x: x.callPackage(), map(lambda x: nameMap[x], sortedSpecs))) )
|
||||
allPackagesFile.close()
|
||||
|
||||
if args.buildRoot != None:
|
||||
buildRootFilename = os.path.normpath( args.buildRoot )
|
||||
if not os.path.exists(os.path.dirname(buildRootFilename)):
|
||||
os.makedirs(os.path.dirname(buildRootFilename))
|
||||
buildRootFile = open(buildRootFilename, 'w')
|
||||
buildRootFile.write( "{ fetchurl, buildRoot }: {\n\n" )
|
||||
keys = buildRootContent.keys()
|
||||
keys.sort()
|
||||
for k in keys:
|
||||
buildRootFile.write( buildRootContent[k] + '\n' )
|
||||
buildRootFile.write( "}\n" )
|
||||
buildRootFile.close()
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user