[CI] Adding assets.json file to the releases. (#9564)

This PR adds a new file to the release: `assets.json`, that offers information about the assets available in the release.

The purpose is to have one persistent link `https://github.com/enso-org/enso/releases/latest/download/assets.json` that has the current download links that can be consumed by the website.

Also, the release template has been updated to use the same assets information source, rather than duplicate the information about artifact names.

Additionally, additional step for release validation was added, so the CI can alert if one of the expected assets is missing.

build/build/release-body.md

@@ -14,11 +14,9 @@ The template itself is written in [Handlebars](https://handlebarsjs.com/).
 Enso IDE is the main product of the Enso project. The packages are stand-alone, they contain both GUI and the backend.
 
 Download links:
-
-- [Linux]({{download_prefix}}/enso-linux-x86_64-{{version}}.AppImage) (AppImage);
-- [macOS (x64)]({{download_prefix}}/enso-mac-x64-{{version}}.dmg) (DMG);
-- [macOS (arm64)]({{download_prefix}}/enso-mac-arm64-{{version}}.dmg) (DMG);
-- [Windows]({{download_prefix}}/enso-win-x64-{{version}}.exe) (Installer Executable).
+{{#each assets.ide}}
+- [{{target_pretty}}]({{url}})
+{{/each}}
 
 This is the recommended download for most users.
 
@@ -27,11 +25,9 @@ This is the recommended download for most users.
 If you are interested in using Enso Engine command line tools only, download the Enso Engine bundle.
 
 Download links:
-
-- [Linux]({{download_prefix}}/enso-bundle-{{version}}-linux-amd64.tar.gz);
-- [macOS (x64)]({{download_prefix}}/enso-bundle-{{version}}-macos-amd64.tar.gz);
-- [macOS (arm64)]({{download_prefix}}/enso-bundle-{{version}}-macos-amd64.tar.gz);
-- [Windows]({{download_prefix}}/enso-bundle-{{version}}-windows-amd64.zip).
+{{#each assets.engine}}
+- [{{target_pretty}}]({{url}})
+{{/each}}
 
 These are archives containing the [Enso portable distribution](https://enso.org/docs/developer/enso/distribution/distribution.html#portable-enso-distribution-layout). User is responsible for setting up the environment variables and adding the `bin` directory to the `PATH`.
 

build/build/src/paths.rs

@@ -86,7 +86,7 @@ impl TargetTriple {
     /// Get the triple effectively used by the Engine build.
     ///
     /// This might differ from `self` if Engine for some reason needs to cross-compile. Currently
-    /// this is not the case, previously it was used to force x64 on Applce Silicon.
+    /// this is not the case, previously it was used to force x64 on Apple Silicon.
     pub fn engine(&self) -> Self {
         self.clone()
     }

build/build/src/project/ide.rs

@@ -43,21 +43,9 @@ impl Artifact {
             _ => todo!("{target_os}-{target_arch} combination is not supported"),
         }
         .into();
-        // Electron-builder does something like this:
-        // https://github.com/electron-userland/electron-builder/blob/master/packages/builder-util/src/arch.ts
-        let arch_string = match (target_os, target_arch) {
-            (OS::Linux, Arch::X86_64) => "x86_64",
-            (_, Arch::X86_64) => "x64",
-            (_, Arch::AArch64) => "arm64",
-            _ => todo!("{target_os}-{target_arch} combination is not supported"),
-        };
-        let image = dist_dir.as_ref().join(match target_os {
-            OS::Linux => format!("enso-linux-{arch_string}-{version}.AppImage"),
-            OS::MacOS => format!("enso-mac-{arch_string}-{version}.dmg"),
-            OS::Windows => format!("enso-win-{arch_string}-{version}.exe"),
-            _ => todo!("{target_os}-{target_arch} combination is not supported"),
-        });
 
+        let image_filename = electron_image_filename(target_os, target_arch, version);
+        let image = dist_dir.as_ref().join(image_filename);
         Self {
             image_checksum: image.with_extension("sha256"),
             image,
@@ -144,3 +132,21 @@ impl Ide {
         .boxed()
     }
 }
+
+/// Filename of the image that electron-builder will produce.
+pub fn electron_image_filename(target_os: OS, target_arch: Arch, version: &Version) -> String {
+    // Electron-builder does something like this:
+    // https://github.com/electron-userland/electron-builder/blob/master/packages/builder-util/src/arch.ts
+    let arch_string = match (target_os, target_arch) {
+        (OS::Linux, Arch::X86_64) => "x86_64",
+        (_, Arch::X86_64) => "x64",
+        (_, Arch::AArch64) => "arm64",
+        _ => todo!("{target_os}-{target_arch} combination is not supported"),
+    };
+    match target_os {
+        OS::Linux => format!("enso-linux-{arch_string}-{version}.AppImage"),
+        OS::MacOS => format!("enso-mac-{arch_string}-{version}.dmg"),
+        OS::Windows => format!("enso-win-{arch_string}-{version}.exe"),
+        _ => todo!("{target_os}-{target_arch} combination is not supported"),
+    }
+}

build/build/src/project_manager.rs

@@ -2,8 +2,6 @@ use crate::prelude::*;
 
 use crate::paths::TargetTriple;
 
-use ide_ci::github::release::ARCHIVE_EXTENSION;
-
 
 
 pub fn url(target: &TargetTriple) -> Result<Url> {
@@ -13,7 +11,7 @@ pub fn url(target: &TargetTriple) -> Result<Url> {
         repo = "ci-build",
         tag = target.versions.tag(),
         asset = format!("project-manager-bundle-{target}"),
-        ext = ARCHIVE_EXTENSION,
+        ext = ide_ci::github::release::archive_extension(),
     );
     Url::parse(&url_text).anyhow_err()
 }

build/build/src/release.rs

@@ -24,6 +24,13 @@ use serde_json::json;
 use tempfile::tempdir;
 
 
+// ==============
+// === Export ===
+// ==============
+
+pub mod manifest;
+
+
 
 /// Get the prefix of URL of the release's asset in GitHub.
 ///
@@ -36,10 +43,22 @@ use tempfile::tempdir;
 /// let repo = RepoRef::new("enso-org", "enso");
 /// let version = Version::from_str("2020.1.1").unwrap();
 /// let prefix = download_asset_prefix(&repo, &version);
-/// assert_eq!(prefix, "https://github.com/enso-org/enso/releases/download/2020.1.1");
+/// assert_eq!(prefix.as_str(), "https://github.com/enso-org/enso/releases/download/2020.1.1");
 /// ```
-pub fn download_asset_prefix(repo: &impl IsRepo, version: &Version) -> String {
-    format!("https://github.com/{repo}/releases/download/{version}",)
+pub fn download_asset_prefix(repo: &impl IsRepo, version: &Version) -> Url {
+    let text = format!("https://github.com/{repo}/releases/download/{version}",);
+    Url::from_str(&text).expect("Failed to parse the URL.")
+}
+
+/// Get the URL for downloading the asset from the GitHub release.
+pub fn download_asset(
+    repo: &impl IsRepo,
+    version: &Version,
+    asset_name: impl AsRef<str>,
+) -> String {
+    let prefix = download_asset_prefix(repo, version);
+    let asset_name = asset_name.as_ref();
+    format!("{prefix}/{asset_name}")
 }
 
 /// Generate placeholders for the release notes.
@@ -55,12 +74,11 @@ pub fn release_body_placeholders(
     ret.insert("edition", context.triple.versions.edition_name().into());
     ret.insert("repo", serde_json::to_value(&context.remote_repo)?);
     ret.insert(
-        "download_prefix",
-        format!(
-            "https://github.com/{}/releases/download/{}",
-            context.remote_repo, context.triple.versions.version
-        )
-        .into(),
+        "assets",
+        serde_json::to_value(manifest::Assets::new(
+            &context.remote_repo,
+            &context.triple.versions.version,
+        ))?,
     );
 
     // Generate the release notes.
@@ -174,6 +192,39 @@ pub async fn publish_release(context: &BuildContext) -> Result {
     debug!("Updating edition in the AWS S3.");
     crate::aws::update_manifest(&remote_repo, &edition_file_path).await?;
 
+    // Add assets manifest.
+    let manifest = manifest::Assets::new(&remote_repo, &triple.versions.version);
+    let tempdir = tempdir()?;
+    let manifest_path = tempdir.path().join(manifest::ASSETS_MANIFEST_FILENAME);
+    ide_ci::fs::write_json(&manifest_path, &manifest)?;
+    release_handle.upload_asset_file(&manifest_path).await?;
+
+    // The validation step is performed to enable issue reporting and enhance issue visibility.
+    // Currently, even if the validation fails, the release will not be retracted.
+    validate_release(release_handle).await?;
+
+    Ok(())
+}
+
+/// Perform basic check if the release contains advertised assets.
+///
+/// This should be run only on a published (non-draft) release, as asset download URLs change after
+/// publishing.
+#[context("Failed to validate release: {release:?}")]
+pub async fn validate_release(release: github::release::Handle) -> Result {
+    let info = release.get().await?;
+    ensure!(!info.draft, "Release is a draft.");
+    let version = Version::from_str(&info.tag_name)?;
+    let manifest_url = download_asset(&release.repo, &version, manifest::ASSETS_MANIFEST_FILENAME);
+    let manifest = ide_ci::io::download_all(&manifest_url)
+        .await
+        .context("Failed to download assets manifest.")?;
+    let manifest: manifest::Assets =
+        serde_json::from_slice(&manifest).context("Failed to parse assets manifest.")?;
+    for asset in manifest.assets() {
+        let response = reqwest::Client::new().get(&asset.url).send().await?;
+        ensure!(response.status().is_success(), "Failed to download asset: {}", asset.url);
+    }
     Ok(())
 }
 
@@ -293,10 +344,11 @@ pub async fn promote_release(context: &BuildContext, version_designation: Design
     Ok(())
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
+    use ide_ci::cache::Cache;
+    use ide_ci::github::setup_octocrab;
 
     #[tokio::test]
     #[ignore]
@@ -306,4 +358,39 @@ mod tests {
         notify_cloud_about_gui(&version).await?;
         Ok(())
     }
+
+    #[tokio::test]
+    #[ignore]
+    async fn release_assets() -> Result {
+        setup_logging()?;
+
+        let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        let repo_root = crate_dir.parent().unwrap().parent().unwrap();
+        let version = Version::from_str("2024.1.1-nightly.2024.3.26")?;
+        let triple = TargetTriple::new(Versions::new(version.clone()));
+        let context = BuildContext {
+            inner:       project::Context {
+                repo_root: crate::paths::new_repo_root(repo_root, &triple),
+                octocrab:  setup_octocrab().await?,
+                cache:     Cache::new_default().await?,
+            },
+            remote_repo: github::Repo::new("enso-org", "enso"),
+            triple:      TargetTriple::new(Versions::new(version.clone())),
+        };
+
+        let release_body = generate_release_body(&context).await?;
+        debug!("Release body: {}", release_body);
+
+        let manifest = manifest::Assets::new(&context.remote_repo, &version);
+        let manifest_json = serde_json::to_string_pretty(&manifest)?;
+        debug!("Manifest: {}", manifest_json);
+
+        let all_assets = manifest.ide.iter().chain(&manifest.engine);
+        for asset in all_assets {
+            let response = reqwest::Client::new().get(&asset.url).send().await?;
+            ensure!(response.status().is_success(), "Failed to download asset: {}", asset.url);
+        }
+
+        Ok(())
+    }
 }

build/build/src/release/manifest.rs

@@ -0,0 +1,92 @@
+//! Description of the release assets.
+//!
+//! See [Assets].
+
+use crate::prelude::*;
+
+use crate::paths::TargetTriple;
+use crate::project;
+use crate::release;
+use crate::version::Versions;
+
+
+
+/// Name of the assets manifest file.
+///
+/// The website uses this name to find the assets manifest, so it should be kept in sync.
+pub const ASSETS_MANIFEST_FILENAME: &str = "assets.json";
+
+/// A platform-specific asset being part of the release, see [Assets] for the purpose.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Asset {
+    pub os:            OS,
+    pub arch:          Arch,
+    pub url:           String,
+    /// User-friendly description of the target platform.
+    pub target_pretty: String,
+}
+
+impl Asset {
+    pub fn new(url: String, triple: &TargetTriple) -> Self {
+        let target_pretty = match (triple.os, triple.arch) {
+            (OS::Windows, Arch::X86_64) => "Windows".into(),
+            (OS::Linux, Arch::X86_64) => "Linux".into(),
+            (OS::MacOS, Arch::X86_64) => "macOS (Intel)".into(),
+            (OS::MacOS, Arch::AArch64) => "macOS (Apple silicon)".into(),
+            (os, arch) => format!("{os} {arch}"),
+        };
+        Self { os: triple.os, arch: triple.arch, url, target_pretty }
+    }
+
+    /// Description od the asset with IDE image.
+    pub fn new_ide(repo: &impl IsRepo, triple: &TargetTriple) -> Self {
+        let filename =
+            project::ide::electron_image_filename(triple.os, triple.arch, &triple.versions.version);
+        let url = release::download_asset(repo, &triple.versions.version, filename);
+        Self::new(url, triple)
+    }
+
+    /// Description od the asset with Engine bundle.
+    pub fn new_engine(repo: &impl IsRepo, triple: &TargetTriple) -> Self {
+        use crate::paths::generated::RepoRootBuiltDistributionEnsoBundleTriple;
+        let stem = RepoRootBuiltDistributionEnsoBundleTriple::segment_name(triple.to_string());
+        let ext = ide_ci::github::release::archive_extension_for(triple.os);
+        let filename = format!("{stem}.{ext}");
+        let url = release::download_asset(repo, &triple.versions.version, filename);
+        Self::new(url, triple)
+    }
+}
+
+
+/// Describes the assets that are part of the release.
+///
+/// The information is used to:
+/// * create `assets.json` file used by the project's website,
+/// * fill information in the release description template.
+// When changing the structure, make sure that it does not break the website.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Assets {
+    /// IDE packages. The exact format (e.g. installer vs AppImage) depends on the platform.
+    pub ide:     Vec<Asset>,
+    /// Engine bundles.
+    pub engine:  Vec<Asset>,
+    /// Version of the release.
+    pub version: Version,
+}
+
+impl Assets {
+    pub fn new(repo: &impl IsRepo, version: &Version) -> Self {
+        let mut ret = Self { ide: vec![], engine: vec![], version: version.clone() };
+        for (os, arch) in crate::ci_gen::RELEASE_TARGETS {
+            let triple = TargetTriple { os, arch, versions: Versions::new(version.clone()) };
+            ret.ide.push(Asset::new_ide(repo, &triple));
+            ret.engine.push(Asset::new_engine(repo, &triple));
+        }
+        ret
+    }
+
+    /// Iterate all the assets described in this manifest.
+    pub fn assets(&self) -> impl Iterator<Item = &Asset> {
+        self.ide.iter().chain(&self.engine)
+    }
+}

build/ci_utils/src/github/release.rs

@@ -20,14 +20,21 @@ pub use octocrab::models::ReleaseId as Id;
 
 
 
-/// The extensions that will be used for the archives in the GitHub release assets.
+/// Extension of the preferred archive format for release assets on the current platform.
+pub fn archive_extension() -> &'static str {
+    archive_extension_for(TARGET_OS)
+}
+
+/// Get archive format extension for release assets targeting the given operating system.
 ///
-/// On Windows we use `.zip`, because it has out-of-the-box support in the Explorer.
-/// On other platforms we use `.tar.gz`, because it is a good default.
-pub const ARCHIVE_EXTENSION: &str = match TARGET_OS {
-    OS::Windows => "zip",
-    _ => "tar.gz",
-};
+/// - For Windows, we use `.zip` because it has built-in support in Windows Explorer.
+/// - For all other operating systems, we use `.tar.gz` as the default.
+pub fn archive_extension_for(os: OS) -> &'static str {
+    match os {
+        OS::Windows => "zip",
+        _ => "tar.gz",
+    }
+}
 
 /// Types that uniquely identify a release and can be used to fetch it from GitHub.
 pub trait IsRelease: Debug {
@@ -155,7 +162,7 @@ pub trait IsReleaseExt: IsRelease + Sync {
         let dir_to_upload = dir_to_upload.as_ref();
         let temp_dir = tempfile::tempdir()?;
         let archive_path =
-            custom_name.with_parent(temp_dir.path()).with_appended_extension(ARCHIVE_EXTENSION);
+            custom_name.with_parent(temp_dir.path()).with_appended_extension(archive_extension());
         crate::archive::create(&archive_path, [&dir_to_upload]).await?;
         self.upload_asset_file(archive_path).await
     }