oxalica/rust-overlay
1
1
Fork 0
mirror of https://github.com/oxalica/rust-overlay.git synced 2024-10-04 05:07:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add support for beta channel

Browse Source
This commit is contained in:
oxalica 2021-01-05 03:38:46 -06:00
parent 164fa885dc
commit a04ffd9804
No known key found for this signature in database
GPG Key ID: CED392DE0C483D00
7 changed files with 134 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@ -11,8 +11,8 @@ so the evaluation is *pure* and no need to have network (but [nixpkgs-mozilla][m
It also works well with [Nix Flakes](https://nixos.wiki/wiki/Flakes).
- The toolchain hashes are auto-updated daily using GitHub Actions.
- Current oldest supported version is stable 1.29.0 and nightly 2018-09-13
(which is randomly chosen).
- Current oldest supported version is stable 1.29.0 and beta/nightly 2018-09-13
(which are randomly chosen).
## Use as a classic Nix overlay
@ -108,6 +108,14 @@ Here's an example of using it in nixos configuration.
# ... other versions.
};
beta = {
# The latest beta toolchain.
latest = { /* toolchain */ };
"2021-01-01" = { /* toolchain */ };
"2020-12-30" = { /* toolchain */ };
# ... other versions.
};
nightly = {
# The latest nightly toolchain.
latest = { /* toolchain */ };
@ -130,10 +138,12 @@ Here's an example of using it in nixos configuration.
Some examples (assume `nixpkgs` had the overlay applied):
- Latest stable rust with all default components:
`nixpkgs.rust-bin.stable.latest.rust`
- Latest nightly rust with all default components:
`nixpkgs.rust-bin.nightly.latest.rust`
- Latest stable/beta/nightly rust with all default components:
`nixpkgs.rust-bin.{stable,beta,nightly}.latest.rust`
- A specific version of stable rust:
`nixpkgs.rust-bin.stable."1.48.0".rust`
- A specific date of beta rust:
`nixpkgs.rust-bin.nightly."2021-01-01".rust`
- A specific version of stable rust:
`nixpkgs.rust-bin.stable."1.48.0".rust`
- A specific date of nightly rust:

131
fetch.py
View File

@ -17,7 +17,7 @@ RETRY_DELAY = 3.0
SYNC_MAX_FETCH = 5
MIN_STABLE_VERSION = '1.29.0'
MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
MIN_BETA_DATE = MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
DIST_ROOT = 'https://static.rust-lang.org/dist'
NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
@ -86,19 +86,24 @@ def compress_renames(renames: dict) -> int:
f.write(']\n')
return idx
def retry_with(f):
def fetch_url(url: str, params=None, allow_not_found=False):
i = 0
while True:
resp = None
try:
return f()
resp = requests.get(url, params=params)
if resp.status_code == 404 and allow_not_found:
return None
resp.raise_for_status()
return resp
except requests.exceptions.RequestException as e:
i += 1
if i >= MAX_TRIES:
if (resp is not None and resp.status_code == 404) or i >= MAX_TRIES:
raise
print(e)
time.sleep(RETRY_DELAY)
def translate_dump_manifest(manifest: str, f, nightly=False):
def translate_dump_manifest(channel: str, manifest: str, f):
manifest = toml.loads(manifest)
date = manifest['date']
rustc_version = manifest['pkg']['rustc']['version'].split()[0]
@ -148,20 +153,25 @@ def translate_dump_manifest(manifest: str, f, nightly=False):
f.write('};')
f.write('}\n')
def fetch_stable_manifest(version: str, out_path: Path):
def fetch_manifest(channel: str, version: str, out_path: Path):
out_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = out_path.with_suffix('.tmp')
print(f'Fetching stable {version}')
manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-{version}.toml'))
manifest.raise_for_status()
print(f'Fetching {channel} {version}')
if channel == 'stable':
url = f'{DIST_ROOT}/channel-rust-{version}.toml'
else:
url = f'{DIST_ROOT}/{version}/channel-rust-{channel}.toml'
manifest = fetch_url(url, allow_not_found=channel != 'stable')
if manifest is None:
print('Not found, skipped')
return
manifest = manifest.text
MANIFEST_TMP_PATH.write_text(manifest)
with open(tmp_path, 'w') as fout:
translate_dump_manifest(manifest, fout)
translate_dump_manifest(channel, manifest, fout)
tmp_path.rename(out_path)
def update_stable_index():
dir = Path('manifests/stable')
def update_stable_index(dir=Path('manifests/stable')):
versions = sorted(
(file.stem for file in dir.iterdir() if file.stem != 'default' and file.suffix == '.nix'),
key=parse_version,
@ -173,6 +183,19 @@ def update_stable_index():
f.write(f' latest = {escape_nix_string(versions[-1])};\n')
f.write('}\n')
def update_beta_index():
update_nightly_index(dir=Path('manifests/beta'))
def update_nightly_index(dir=Path('manifests/nightly')):
dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
with open(str(dir / 'default.nix'), 'w') as f:
f.write('{\n')
for date in dates:
year = date.split('-')[0]
f.write(f' {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
f.write(f' latest = {escape_nix_string(dates[-1])};\n')
f.write('}\n')
def sync_stable_channel(*, stop_if_exists, max_fetch=None):
GITHUB_RELEASES_URL = 'https://api.github.com/repos/rust-lang/rust/releases'
PER_PAGE = 100
@ -182,12 +205,10 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
while True:
page += 1
print(f'Fetching release page {page}')
release_page = retry_with(lambda: requests.get(
release_page = fetch_url(
GITHUB_RELEASES_URL,
params={'per_page': PER_PAGE, 'page': page},
))
release_page.raise_for_status()
release_page = release_page.json()
).json()
versions.extend(
tag['tag_name']
for tag in release_page
@ -208,66 +229,65 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
continue
print(f'{version} is already fetched. Stopped')
break
fetch_stable_manifest(version, out_path)
fetch_manifest('stable', version, out_path)
processed += 1
assert max_fetch is None or processed <= max_fetch, 'Too many versions'
update_stable_index()
def fetch_nightly_manifest(date: str, out_path: Path):
out_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = out_path.with_suffix('.tmp')
print(f'Fetching nightly {date}')
manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/{date}/channel-rust-nightly.toml'))
if manifest.status_code == 404:
print(f'Not found, skipped')
return
manifest.raise_for_status()
manifest = manifest.text
MANIFEST_TMP_PATH.write_text(manifest)
with open(tmp_path, 'w') as fout:
translate_dump_manifest(manifest, fout, nightly=True)
tmp_path.rename(out_path)
def sync_beta_channel(*, stop_if_exists, max_fetch=None):
# Fetch the global nightly manifest to retrive the latest nightly version.
print('Fetching latest beta version')
manifest = fetch_url(f'{DIST_ROOT}/channel-rust-beta.toml').text
date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
print(f'The latest beta version is {date}')
processed = 0
date += datetime.timedelta(days=1)
while date > MIN_BETA_DATE:
date -= datetime.timedelta(days=1)
date_str = date.isoformat()
out_path = Path(f'manifests/beta/{date.year}/{date_str}.nix')
if out_path.exists():
if not stop_if_exists:
continue
print(f'{date_str} is already fetched. Stopped')
break
fetch_manifest('beta', date_str, out_path)
processed += 1
assert max_fetch is None or processed <= max_fetch, 'Too many versions'
update_beta_index()
def sync_nightly_channel(*, stop_if_exists, max_fetch=None):
# Fetch the global nightly manifest to retrive the latest nightly version.
print('Fetching latest nightly version')
manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-nightly.toml'))
manifest.raise_for_status()
date = datetime.date.fromisoformat(toml.loads(manifest.text)['date'])
manifest = fetch_url(f'{DIST_ROOT}/channel-rust-nightly.toml').text
date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
print(f'The latest nightly version is {date}')
processed = 0
date += datetime.timedelta(days=1)
while date > MIN_NIGHTLY_DATE:
date -= datetime.timedelta(days=1)
out_path = Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix')
date_str = date.isoformat()
out_path = Path(f'manifests/nightly/{date.year}/{date_str}.nix')
if out_path.exists():
if not stop_if_exists:
continue
print(f'{date} is already fetched. Stopped')
print(f'{date_str} is already fetched. Stopped')
break
fetch_nightly_manifest(date.isoformat(), out_path)
fetch_manifest('nightly', date_str, out_path)
processed += 1
assert max_fetch is None or processed <= max_fetch, 'Too many versions'
update_nightly_index()
def update_nightly_index():
dir = Path('manifests/nightly')
dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
with open(str(dir / 'default.nix'), 'w') as f:
f.write('{\n')
for date in dates:
year = date.split('-')[0]
f.write(f' {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
f.write(f' latest = {escape_nix_string(dates[-1])};\n')
f.write('}\n')
def main():
args = sys.argv[1:]
if len(args) == 0:
print('Synchronizing stable channels')
sync_stable_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
print('Synchronizing nightly channels')
print('\nSynchronizing beta channels')
sync_beta_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
print('\nSynchronizing nightly channels')
sync_nightly_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
elif len(args) == 2 and args[0] == 'stable':
if args[1] == 'all':
@ -275,14 +295,23 @@ def main():
else:
version = args[1]
assert RE_STABLE_VERSION.match(version), 'Invalid version'
fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
fetch_manifest('stable', version, Path(f'manifests/stable/{version}.nix'))
update_stable_index()
elif len(args) == 2 and args[0] == 'beta':
if args[1] == 'all':
sync_beta_channel(stop_if_exists=False)
else:
date = datetime.date.fromisoformat(args[1])
date_str = date.isoformat()
fetch_manifest('beta', date_str, Path(f'manifests/beta/{date.year}/{date_str}.nix'))
update_beta_index()
elif len(args) == 2 and args[0] == 'nightly':
if args[1] == 'all':
sync_nightly_channel(stop_if_exists=False)
else:
date = datetime.date.fromisoformat(args[1])
fetch_nightly_manifest(date, Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix'))
date_str = date.isoformat()
fetch_manifest('nightly', date_str, Path(f'manifests/nightly/{date.year}/{date_str}.nix'))
update_nightly_index()
else:
print('''

8
flake.nix
View File

@ -53,7 +53,7 @@
checks = let
inherit (pkgs) rust-bin rustChannelOf;
inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable nightly;
inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable beta nightly;
rustTarget = pkgs.rust.toRustTarget pkgs.hostPlatform;
@ -71,6 +71,7 @@
url-kind-0 = assertUrl stable."1.47.0".cargo "https://static.rust-lang.org/dist/2020-10-08/cargo-0.48.0-${rustTarget}.tar.xz";
url-kind-1 = assertUrl stable."1.34.2".llvm-tools-preview "https://static.rust-lang.org/dist/2019-05-14/llvm-tools-1.34.2%20(6c2484dc3%202019-05-13)-${rustTarget}.tar.xz";
url-kind-nightly = assertUrl nightly."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-nightly-${rustTarget}.tar.xz";
url-kind-beta = assertUrl beta."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-beta-${rustTarget}.tar.xz";
url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz";
rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview;
@ -80,17 +81,22 @@
};
latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust;
latest-beta = assertEq pkgs.latest.rustChannels.beta.rust beta.latest.rust;
latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust;
rust-channel-of-stable = assertEq (rustChannelOf { channel = "stable"; }).rust stable.latest.rust;
rust-channel-of-beta = assertEq (rustChannelOf { channel = "beta"; }).rust beta.latest.rust;
rust-channel-of-nightly = assertEq (rustChannelOf { channel = "nightly"; }).rust nightly.latest.rust;
rust-channel-of-version = assertEq (rustChannelOf { channel = "1.48.0"; }).rust stable."1.48.0".rust;
rust-channel-of-nightly-date = assertEq (rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rust nightly."2021-01-01".rust;
rust-channel-of-beta-date = assertEq (rustChannelOf { channel = "beta"; date = "2021-01-01"; }).rust beta."2021-01-01".rust;
rustup-toolchain-stable = assertEq (fromRustupToolchain { channel = "stable"; }) stable.latest.rust;
rustup-toolchain-beta = assertEq (fromRustupToolchain { channel = "beta"; }) beta.latest.rust;
rustup-toolchain-nightly = assertEq (fromRustupToolchain { channel = "nightly"; }) nightly.latest.rust;
rustup-toolchain-version = assertEq (fromRustupToolchain { channel = "1.48.0"; }) stable."1.48.0".rust;
rustup-toolchain-nightly-date = assertEq (fromRustupToolchain { channel = "nightly-2021-01-01"; }) nightly."2021-01-01".rust;
rustup-toolchain-beta-date = assertEq (fromRustupToolchain { channel = "beta-2021-01-01"; }) beta."2021-01-01".rust;
rustup-toolchain-customization = assertEq
(fromRustupToolchain {
channel = "1.48.0";

17
manifest.nix
View File

@ -42,7 +42,7 @@ let
# Uncompress the compressed manifest to the original one
# (not complete but has enough information to make up the toolchain).
uncompressManifest = nightly: version: {
uncompressManifest = channel: version: {
v, # rustc version
d, # date
r, # rename index
@ -61,9 +61,9 @@ let
pkgNameStripped = removeSuffix "-preview" pkgName;
targetTail = if targetIdx == "_" then "" else "-" + target;
urlVersion =
if u != null then u # Use specified url version if exists.
else if nightly then "nightly" # Otherwise, for nightly channel, default to be "nightly".
else v; # For stable channel, default to be rustc version.
if u != null then u # Use specified url version if exists.
else if channel == "stable" then v # For stable channel, default to be rustc version.
else channel; # Otherwise, for beta/nightly channel, default to be "beta"/"nightly".
in {
name = target;
value = {
@ -74,8 +74,8 @@ let
}) (removeAttrs manifest ["v" "d" "r"]);
};
uncompressManifestSet = nightly: set: let
ret = mapAttrs (uncompressManifest nightly) (removeAttrs set ["latest"]);
uncompressManifestSet = channel: set: let
ret = mapAttrs (uncompressManifest channel) (removeAttrs set ["latest"]);
in ret // { latest = ret.${set.latest}; };
in {
@ -86,8 +86,9 @@ in {
# For internal usage.
manifests = {
stable = uncompressManifestSet false (import ./manifests/stable);
nightly = uncompressManifestSet true (import ./manifests/nightly);
stable = uncompressManifestSet "stable" (import ./manifests/stable);
beta = uncompressManifestSet "beta" (import ./manifests/beta);
nightly = uncompressManifestSet "nightly" (import ./manifests/nightly);
};
};
}

1
manifests/beta/2021/2021-01-01.nix Normal file
View File

File diff suppressed because one or more lines are too long

4
manifests/beta/default.nix Normal file
View File

@ -0,0 +1,4 @@
{
"2021-01-01" = import ./2021/2021-01-01.nix;
latest = "2021-01-01";
}

19
rust-overlay.nix
View File

@ -16,21 +16,33 @@ let
asVersion = match "[0-9]+\\.[0-9]+\\.[0-9]+" channel;
asNightlyDate = let m = match "nightly-([0-9]+-[0-9]+-[0-9]+)" channel; in
if m == null then null else elemAt m 0;
asBetaDate = let m = match "beta-([0-9]+-[0-9]+-[0-9]+)" channel; in
if m == null then null else elemAt m 0;
in
# "stable"
if channel == "stable" then
assertWith (date == null) "Stable version with specific date is not supported"
manifests.stable.latest
# "nightly"
else if channel == "nightly" then
manifests.nightly.${if date != null then date else "latest"} or (throw "Nightly ${date} is not available")
# "beta"
else if channel == "beta" then
throw "Beta channel is not supported yet"
manifests.beta.${if date != null then date else "latest"} or (throw "Beta ${date} is not available")
# "1.49.0"
else if asVersion != null then
assertWith (date == null) "Stable version with specific date is not supported"
manifests.stable.${channel} or (throw "Stable ${channel} is not available")
# "beta-2021-01-01"
else if asBetaDate != null then
assertWith (date == null) "Cannot specify date in both `channel` and `date`"
manifests.beta.${asBetaDate} or (throw "Beta ${asBetaDate} is not available")
# "nightly-2021-01-01"
else if asNightlyDate != null then
assertWith (date == null) "Cannot specify date in both `channel` and `date`"
manifests.nightly.${asNightlyDate} or (throw "Nightly ${asNightlyDate} is not available")
# Otherwise
else throw "Unknown channel: ${channel}";
# Select a toolchain and aggregate components by rustup's `rust-toolchain` file format.
@ -340,6 +352,9 @@ in {
# For a specific version of stable:
# rust-bin.stable."1.47.0".rust
#
# For a specific date of beta:
# rust-bin.beta."2021-01-01".rust
#
# For a specific date of nightly:
# rust-bin.nightly."2020-01-01".rust
rust-bin = with builtins;
@ -376,7 +391,7 @@ in {
latest = (super.latest or {}) // {
rustChannels = {
stable = self.rust-bin.stable.latest;
beta = throw "Beta channel is not supported yet";
beta = self.rust-bin.beta.latest;
nightly = self.rust-bin.nightly.latest;
};
};