Add nightly support

all.nix

@@ -0,0 +1,11 @@
+# For test.
+with import <nixpkgs> {
+  overlays = [ (import ./.) ];
+}; {
+  stable = lib.mapAttrs
+    (channel: _: (rustChannelOf { inherit channel; }).rust)
+    (removeAttrs (import ./manifests/stable) ["latest"]);
+  nightly = lib.mapAttrs
+    (date: _: (rustChannelOf { channel = "nightly"; inherit date; }).rust)
+    (removeAttrs (import ./manifests/nightly) ["latest"]);
+}

default.nix

@@ -40,7 +40,7 @@ let
   };
 
   # version -> { pkgName = { _1 = "..."; } } -> { pkgName = { x86_64-unknown-linux-gnu = fetchurl { .. }; } }
-  uncompressManifest = version: { date, ... }@manifest: rec {
+  uncompressManifest = nightly: version: { date, ... }@manifest: rec {
     inherit date;
     pkg =
       mapAttrs (pkgName: { v, k ? 0, ... }@hashes: {
@@ -50,10 +50,12 @@ let
             target = targets.${targetIdx};
             pkgNameStripped = removeSuffix "-preview" pkgName;
             targetTail = if targetIdx == "_" then "" else "-" + target;
+            vHead = head (match "([^ ]*) .*" v);
             urlVersion =
-              if k == 0 then head (match "([^ ]*) .*" v) # '0.44.1 (aaaaaaaaa 2018-01-01)' -> '0.44.1' [package version]
-              else if k == 1 then v                      # '0.44.1 (aaaaaaaaa 2018-01-01)' [package version]
-              else if k == 2 then version                # '1.49.0' [stable toolchain version]
+              if nightly then "nightly"     # 'nightly'
+              else if k == 0 then vHead     # '0.44.1 (aaaaaaaaa 2018-01-01)' -> '0.44.1' [package version]
+              else if k == 1 then v         # '0.44.1 (aaaaaaaaa 2018-01-01)' [package version]
+              else if k == 2 then version   # '1.49.0' [stable toolchain version]
               else throw "Invalid k";
           in {
             name = target;
@@ -65,12 +67,13 @@ let
       }) (removeAttrs manifest ["date"]);
   };
 
-  uncompressManifestSet = set: let
-    ret = mapAttrs uncompressManifest (removeAttrs set ["latest"]);
+  uncompressManifestSet = nightly: set: let
+    ret = mapAttrs (uncompressManifest nightly) (removeAttrs set ["latest"]);
   in ret // { latest = ret.${set.latest}; };
 
   manifests = {
-    stable = uncompressManifestSet (import ./manifests/stable);
+    stable = uncompressManifestSet false (import ./manifests/stable);
+    nightly = uncompressManifestSet true (import ./manifests/nightly);
   };
 
 # in { inherit manifests; }

fetch.py

@@ -6,6 +6,7 @@ import re
 import string
 import sys
 import time
+import datetime
 from pathlib import Path
 
 import toml
@@ -16,8 +17,9 @@ RETRY_DELAY = 3.0
 SYNC_MAX_FETCH = 5
 
 MIN_STABLE_VERSION = '1.29.0'
+MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
 
-DIST_SERVER = 'https://static.rust-lang.org'
+DIST_ROOT = 'https://static.rust-lang.org/dist'
 NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
 MANIFEST_TMP_PATH = Path('manifest.tmp')
 TARGETS_PATH = Path('manifests/targets.nix')
@@ -76,7 +78,7 @@ def retry_with(f):
             print(e)
             time.sleep(RETRY_DELAY)
 
-def translate_dump_manifest(manifest: str, f):
+def translate_dump_manifest(manifest: str, f, nightly=False):
     manifest = toml.loads(manifest)
     date = manifest['date']
     version = manifest['pkg']['rustc']['version'].split()[0]
@@ -96,19 +98,26 @@ def translate_dump_manifest(manifest: str, f):
                 continue
             url = target['xz_url']
             target_tail = '' if target_name == '*' else '-' + target_name
-            start = f'https://static.rust-lang.org/dist/{date}/{pkg_name_stripped}-'
+            start = f'{DIST_ROOT}/{date}/{pkg_name_stripped}-'
             end = f'{target_tail}.tar.xz'
+            # Occurs in nightly-2019-01-10. Maybe broken or hirarerchy change?
+            if url.startswith('nightly/'):
+                url = DIST_ROOT + url[7:]
             assert url.startswith(start) and url.endswith(end), f'Unexpected url: {url}'
             url_version = url[len(start):-len(end)]
 
-        if url_version == pkg['version'].split(' ')[0]:
+        if url_version == 'nightly':
+            assert nightly
+            url_version_kind = 0
+        elif url_version == pkg['version'].split(' ')[0]:
+            assert not nightly
             url_version_kind = 0
         elif url_version == pkg['version']:
             url_version_kind = 1
         elif url_version == version:
             url_version_kind = 2
         else:
-            assert False, f'Known url version `{url_version}` for `{pkg_name}`'
+            assert False, f'Unknow url version `{url_version}` for `{pkg_name}`'
 
         f.write(f'{pkg_name}={{')
         f.write(f'v={escape_nix_string(pkg["version"])};')
@@ -119,6 +128,9 @@ def translate_dump_manifest(manifest: str, f):
             if not target['available']:
                 continue
             url = target['xz_url']
+            # See above.
+            if url.startswith('nightly/'):
+                url = DIST_ROOT + url[7:]
             hash = to_base64(target['xz_hash']) # Hash must not contains quotes.
             target_tail = '' if target_name == '*' else '-' + target_name
             expect_url = f'https://static.rust-lang.org/dist/{date}/{pkg_name_stripped}-{url_version}{target_tail}.tar.xz'
@@ -128,9 +140,10 @@ def translate_dump_manifest(manifest: str, f):
     f.write('}\n')
 
 def fetch_stable_manifest(version: str, out_path: Path):
+    out_path.parent.mkdir(parents=True, exist_ok=True)
     tmp_path = out_path.with_suffix('.tmp')
-    print(f'Fetching {version}')
-    manifest = retry_with(lambda: requests.get(f'{DIST_SERVER}/dist/channel-rust-{version}.toml'))
+    print(f'Fetching stable {version}')
+    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-{version}.toml'))
     manifest.raise_for_status()
     manifest = manifest.text
     MANIFEST_TMP_PATH.write_text(manifest)
@@ -191,30 +204,83 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
         assert max_fetch is None or processed <= max_fetch, 'Too many versions'
     update_stable_index()
 
-def init_sync_all():
-    sync_stable_channel(stop_if_exists=False)
+def fetch_nightly_manifest(date: str, out_path: Path):
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = out_path.with_suffix('.tmp')
+    print(f'Fetching nightly {date}')
+    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/{date}/channel-rust-nightly.toml'))
+    if manifest.status_code == 404:
+        print(f'Not found, skipped')
+        return
+    manifest.raise_for_status()
+    manifest = manifest.text
+    MANIFEST_TMP_PATH.write_text(manifest)
+    with open(tmp_path, 'w') as fout:
+        translate_dump_manifest(manifest, fout, nightly=True)
+    tmp_path.rename(out_path)
+
+def sync_nightly_channel(*, stop_if_exists, max_fetch=None):
+    # Fetch the global nightly manifest to retrive the latest nightly version.
+    print('Fetching latest nightly version')
+    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-nightly.toml'))
+    manifest.raise_for_status()
+    date = datetime.date.fromisoformat(toml.loads(manifest.text)['date'])
+    print(f'The latest nightly version is {date}')
+
+    processed = 0
+    date + datetime.timedelta(days=1)
+    while date > MIN_NIGHTLY_DATE:
+        date -= datetime.timedelta(days=1)
+        out_path = Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix')
+        if out_path.exists():
+            if not stop_if_exists:
+                continue
+            print('Stopped on existing version')
+            break
+        fetch_nightly_manifest(date.isoformat(), out_path)
+        processed += 1
+        assert max_fetch is None or processed <= max_fetch, 'Too many versions'
+    update_nightly_index()
+
+def update_nightly_index():
+    dir = Path('manifests/nightly')
+    dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
+    with open(str(dir / 'default.nix'), 'w') as f:
+        f.write('{\n')
+        for date in dates:
+            year = date.split('-')[0]
+            f.write(f'  {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
+        f.write(f'  latest = {escape_nix_string(dates[-1])};\n')
+        f.write('}\n')
 
 def main():
     args = sys.argv[1:]
     if len(args) == 0:
         print('Synchronizing channels')
         sync_stable_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
-    elif len(args) == 1:
-        if args[0] == 'all':
+    elif len(args) == 2 and args[0] == 'stable':
+        if args[1] == 'all':
             sync_stable_channel(stop_if_exists=False)
         else:
-            version = args[0]
+            version = args[1]
             assert RE_STABLE_VERSION.match(version), 'Invalid version'
             fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
             update_stable_index()
+    elif len(args) == 2 and args[0] == 'nightly':
+        if args[1] == 'all':
+            sync_nightly_channel(stop_if_exists=False)
+        else:
+            date = datetime.date.fromisoformat(args[1])
+            fetch_nightly_manifest(date, Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix'))
+            update_nightly_index()
     else:
         print('''
 Usage:
     {0}
         Auto-sync new versions from channels.
-    {0} <version>
-        Force to fetch a specific version.
-    {0} all
+    {0} <channel> <version>
+        Force to fetch a specific version from a channel.
+    {0} <channel> all
         Force to fetch all versions.
 '''.format(sys.argv[0]))
         exit(1)

flake.nix

@@ -35,12 +35,15 @@
     defaultPackage = packages.rust-stable;
     packages = {
       rust-stable = pkgs.latest.rustChannels.stable.rust;
+      rust-nightly = pkgs.latest.rustChannels.nightly.rust;
     };
 
     checks = {
       kind2 = (pkgs.rustChannelOf { channel = "1.48.0"; }).rust;
       kind0 = (pkgs.rustChannelOf { channel = "1.47.0"; }).rust;
       kind1 = (pkgs.rustChannelOf { channel = "1.34.2"; }).llvm-tools-preview;
+      kind-nightly = (pkgs.rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rust;
+      url-fix = (pkgs.rustChannelOf { channel = "nightly"; date = "2019-01-10"; }).rust;
     };
   });
 }

manifests/nightly/default.nix

@@ -0,0 +1,8 @@
+{
+  "2020-12-28" = import ./2020/2020-12-28.nix;
+  "2020-12-29" = import ./2020/2020-12-29.nix;
+  "2020-12-30" = import ./2020/2020-12-30.nix;
+  "2020-12-31" = import ./2020/2020-12-31.nix;
+  "2021-01-01" = import ./2021/2021-01-01.nix;
+  latest = "2021-01-01";
+}

rust-overlay.nix

@@ -8,16 +8,21 @@ manifests:
 let
 
   # Manifest selector.
-  fromManifest = { channel }: { stdenv, fetchurl, patchelf }: let
-    inherit (builtins) match;
-    byVersion = match "([0-9]+\\.[0-9]+\\.[0-9]+)" channel != null;
+  fromManifest = { channel ? null, date ? null }: { stdenv, fetchurl, patchelf }: let
+    assertWith = cond: msg: body: if cond then body else throw msg;
 
-    manifest =
-      if channel == "stable" then manifests.stable.latest
-      else if byVersion then manifests.stable.${channel} or (throw "Version ${channel} is not available")
+    ret =
+      if channel == "stable" then
+        assertWith (date == null) "Stable version with specific date is not supported"
+          manifests.stable.latest
+      else if channel == "nightly" then
+        manifests.nightly.${if date != null then date else "latest"} or (throw "nightly ${date} is not available")
+      else if builtins.match "([0-9]+\\.[0-9]+\\.[0-9]+)" channel != null then
+        assertWith (date == null) "Stable version with specific date is not supported"
+          manifests.stable.${channel} or (throw "Stable ${channel} is not available")
       else throw "Unknown channel: ${channel}";
 
-  in fromManifestFile manifest { inherit stdenv fetchurl patchelf; };
+  in fromManifestFile ret { inherit stdenv fetchurl patchelf; };
 
   getComponentsWithFixedPlatform = pkgs: pkgname: stdenv:
     let