feature: remove cyclic dependencies

notes/node2nix.md

@@ -0,0 +1,12 @@
+## problems with original node2nix implementation
+
+### Bad caching
+- packages are all unpacked at once inside a build instead of in individual derivations
+- packages are unpackged several times in different directories (symlinks could be used)
+
+### Bad build performance
+ - unpacking is done sequentially
+ - pinpointing deps is done sequentially
+
+### build time dependencies unavailable
+ - packages are not available during build (could be fixed by setting NODE_PATH and installing in correct order)

specifications/generic-lock-example.json

@@ -19,6 +19,7 @@
     "translatedBy": "python.pure.poetry",
     "translatorParams": "--flag_application",
     "mainPackage": "requests",
+    "dependencyCyclesRemoved": true,
     "dependencyGraph": {
       "requests": [
         "certifi"

src/apps/cli.py

@@ -6,6 +6,8 @@ import subprocess as sp
 import sys
 import tempfile
 
+import networkx as nx
+
 
 with open (os.environ.get("translatorsJsonFile")) as f:
   translators = json.load(f)
@@ -167,14 +169,38 @@ def translate(args):
 
   # clean up dependency graph
   # remove empty entries
+  depGraph = lock['generic']['dependencyGraph']
   if 'dependencyGraph' in lock['generic']:
-    for pname, deps in lock['generic']['dependencyGraph'].copy().items():
+    for pname, deps in depGraph.copy().items():
       if not deps:
-        del lock['generic']['dependencyGraph'][pname]
-  
-  # re-write dream.lock
-  with open(output, 'w') as f:
-    json.dump(lock, f, indent=2)
+        del depGraph[pname]
+
+  # remove cyclic dependencies
+  edges = set()
+  for pname, deps in depGraph.items():
+    for dep in deps:
+      edges.add((pname, dep))
+  G = nx.DiGraph(sorted(list(edges)))
+  cycle_count = 0
+  removed_edges = []
+  for pname in list(depGraph.keys()):
+    try:
+      while True:
+        cycle = nx.find_cycle(G, pname)
+        cycle_count += 1
+        # remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
+        node_from, node_to = cycle[-1][0], cycle[-1][1]
+        G.remove_edge(node_from, node_to)
+        depGraph[node_from].remove(node_to)
+        removed_edges.append((node_from, node_to))
+    except nx.NetworkXNoCycle:
+      continue
+  if removed_edges:
+    removed_cycles_text = 'Removed Cyclic dependencies:'
+    for node, removed_node in removed_edges:
+      removed_cycles_text += f"\n  {node} -> {removed_node}"
+    print(removed_cycles_text)
+  lock['generic']['dependencyCyclesRemoved'] = True
 
   # calculate combined hash if --combined was specified
   if args.combined:
@@ -207,9 +233,10 @@ def translate(args):
 
     # store the hash in the lock
     lock['generic']['sourcesCombinedHash'] = hash
-    with open(output, 'w') as f:
-      json.dump(order_dict(lock), f, indent=2)
-    
+  
+  # re-write dream.lock
+  with open(output, 'w') as f:
+    json.dump(order_dict(lock), f, indent=2)
 
   print(f"Created {output}")
 

src/apps/default.nix

@@ -8,17 +8,19 @@
 }:
 let
   callPackage = pkgs.callPackage;
+
+  cliPython = pkgs.python3.withPackages (ps: [ ps.networkx ]);
 in
 {
 
   # the unified translator cli
-  cli = callPackage ({ python3, writeScript, ... }:
+  cli = callPackage ({ writeScript, ... }:
     writeScript "cli" ''
       export d2nExternalSources=${externalSources}
 
       translatorsJsonFile=${translators.translatorsJsonFile} \
       dream2nixSrc=${../.} \
-        ${python3}/bin/python ${./cli.py} "$@"
+        ${cliPython}/bin/python ${./cli.py} "$@"
     ''
   ) {};