From d17d1ed4846aa27c9fe773be73f04278d4ae5119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wawrzyniec=20Urba=C5=84czyk?= Date: Mon, 11 Mar 2024 18:58:46 +0100 Subject: [PATCH] [CI] Upload the directory as an artifact, not just its content (#9323) --- build/ci_utils/src/actions/artifacts.rs | 61 +++++++++++++++---- .../ci_utils/src/actions/artifacts/upload.rs | 27 ++++++++ 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/build/ci_utils/src/actions/artifacts.rs b/build/ci_utils/src/actions/artifacts.rs index be5924c7e1..8cff901be5 100644 --- a/build/ci_utils/src/actions/artifacts.rs +++ b/build/ci_utils/src/actions/artifacts.rs @@ -138,14 +138,61 @@ pub fn single_file_provider( Ok(futures::stream::iter([file])) } +/// Scan recursively directory with its subtree for files to upload to the artifact. +/// +/// The directory name will be preserved in the artifact container. +/// +/// # Example +/// ``` +/// # use ide_ci::prelude::*; +/// # use tempfile::TempDir; +/// # use ide_ci::actions::artifacts::single_dir_provider; +/// # use std::path::Path; +/// +/// # #[tokio::main] +/// # async fn main() -> Result { +/// // temp_dir/ +/// // ├── sibling/ +/// // │ └── file +/// // └── uploaded_dir/ +/// // ├── file +/// // └── subdir/ +/// // └── nested_file +/// let dir = TempDir::new()?; +/// let sibling = dir.path().join("sibling"); +/// let file_in_sibling = sibling.join("file"); +/// let uploaded = dir.path().join("uploaded_dir"); +/// let file1 = uploaded.join("file"); +/// let file2 = uploaded.join_iter(["subdir/nested_file"]); +/// ide_ci::fs::create_dir_all(&uploaded)?; +/// ide_ci::fs::create(&file1)?; +/// ide_ci::fs::create(&file2)?; +/// ide_ci::fs::create_dir_all(&sibling)?; +/// ide_ci::fs::create(&file_in_sibling)?; +/// +/// let stream = single_dir_provider(&uploaded)?; +/// let mut found_files = stream.collect::>().await; +/// // Make discovery order irrelevant. +/// found_files.sort_by(|a, b| a.local_path.cmp(&b.local_path)); +/// +/// assert_eq!(found_files.len(), 2); +/// assert_eq!(found_files[0].local_path, file1); +/// assert_eq!(found_files[0].remote_path, Path::new("uploaded_dir/file")); +/// assert_eq!(found_files[1].local_path, file2); +/// assert_eq!(found_files[1].remote_path, Path::new("uploaded_dir/subdir/nested_file")); +/// // Note that sibling directory has not been included. +/// # Ok(()) +/// # } +/// ``` pub fn single_dir_provider(path: &Path) -> Result + 'static> { // TODO not optimal, could discover files at the same time as handling them. + let parent_path = path.try_parent()?; let files = walkdir::WalkDir::new(path) .into_iter() .try_collect_vec()? .into_iter() .filter(|entry| !entry.file_type().is_dir()) - .map(|entry| FileToUpload::new_relative(path, entry.path())) + .map(|entry| FileToUpload::new_relative(parent_path, entry.path())) .try_collect_vec()?; info!("Discovered {} files under the {}.", files.len(), path.display()); @@ -190,7 +237,6 @@ mod tests { use super::*; use crate::actions::artifacts::models::CreateArtifactResponse; use reqwest::StatusCode; - use tempfile::TempDir; use wiremock::matchers::method; use wiremock::Mock; use wiremock::MockServer; @@ -233,17 +279,6 @@ mod tests { //let client = reqwest::Client::builder().default_headers(). } - #[tokio::test] - async fn discover_files_in_dir() -> Result { - let dir = TempDir::new()?; - crate::fs::create(dir.join_iter(["file"]))?; - crate::fs::create(dir.join_iter(["subdir/nested_file"]))?; - let stream = single_dir_provider(dir.as_ref())?; - let v = stream.collect::>().await; - dbg!(v); - Ok(()) - } - #[test] fn deserialize_response() -> Result { let text = r#"{"containerId":11099678,"size":-1,"signedContent":null,"fileContainerResourceUrl":"https://pipelines.actions.githubusercontent.com/VYS7uSE1JB12MkavBOHvD6nounefzg1s5vHmQvfbiLmuvFuM6c/_apis/resources/Containers/11099678","type":"actions_storage","name":"SomeFile","url":"https://pipelines.actions.githubusercontent.com/VYS7uSE1JB12MkavBOHvD6nounefzg1s5vHmQvfbiLmuvFuM6c/_apis/pipelines/1/runs/75/artifacts?artifactName=SomeFile","expiresOn":"2022-01-29T04:07:24.5807079Z","items":null}"#; diff --git a/build/ci_utils/src/actions/artifacts/upload.rs b/build/ci_utils/src/actions/artifacts/upload.rs index 291996bc15..be3c778b46 100644 --- a/build/ci_utils/src/actions/artifacts/upload.rs +++ b/build/ci_utils/src/actions/artifacts/upload.rs @@ -215,6 +215,20 @@ pub struct FileToUpload { } impl FileToUpload { + /// Create a new file to upload. + /// + /// The file will be uploaded directly to the root of the artifact container. + /// + /// # Example + /// ``` + /// use ide_ci::actions::artifacts::upload::FileToUpload; + /// use std::path::Path; + /// + /// let local_path = Path::new("/home/user/uploaded/file"); + /// let file = FileToUpload::new_in_root(local_path).unwrap(); + /// assert_eq!(file.local_path, local_path); + /// assert_eq!(file.remote_path, Path::new("file")); + /// ``` pub fn new_in_root(path: impl Into) -> Result { let local_path = path.into(); let remote_path = local_path.file_name().map(into).ok_or_else(|| { @@ -223,6 +237,19 @@ impl FileToUpload { Ok(Self { local_path, remote_path }) } + /// Create a new file to upload with a relative path. + /// + /// # Example + /// ``` + /// use ide_ci::actions::artifacts::upload::FileToUpload; + /// use std::path::Path; + /// + /// let root_path = Path::new("/home/user"); + /// let local_path = Path::new("/home/user/uploaded/subdir/file"); + /// let file = FileToUpload::new_relative(&root_path, &local_path).unwrap(); + /// assert_eq!(file.local_path, local_path); + /// assert_eq!(file.remote_path, Path::new("uploaded/subdir/file")); + /// ``` pub fn new_relative( root_path: impl AsRef, local_path: impl Into,