zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-12-31 01:24:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Paginate through extensions from the blob store (#12614)

Browse Source 
This PR updates the background task used to fetch extensions from the
blob store to account for the possibility that the result set will be
paginated.

Will now paginate through all of the results and collect them up before
proceeding to determining which extensions need to be synced.

Release Notes:

- N/A
This commit is contained in:
Marshall Bowers 2024-06-03 17:17:46 -04:00 committed by GitHub
parent 14c2fab8ab
commit afc0650a49
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 45 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
crates/collab/src/api/extensions.rs
View File

@ -239,16 +239,20 @@ async fn fetch_extensions_from_blob_store(
) -> anyhow::Result<()> {
log::info!("fetching extensions from blob store");
let mut next_marker = None;
let mut published_versions = HashMap::<String, Vec<String>>::default();
loop {
let list = blob_store_client
.list_objects()
.bucket(blob_store_bucket)
.prefix("extensions/")
.set_marker(next_marker.clone())
.send()
.await?;
let objects = list.contents.unwrap_or_default();
log::info!("fetched {} object(s) from blob store", objects.len());
let mut published_versions = HashMap::<&str, Vec<&str>>::default();
for object in &objects {
let Some(key) = object.key.as_ref() else {
continue;
@ -265,35 +269,44 @@ async fn fetch_extensions_from_blob_store(
};
if parts.next() == Some("manifest.json") {
published_versions
.entry(extension_id)
.entry(extension_id.to_owned())
.or_default()
.push(version);
.push(version.to_owned());
}
}
if let (Some(true), Some(last_object)) = (list.is_truncated, objects.last()) {
next_marker.clone_from(&last_object.key);
} else {
break;
}
}
log::info!("found {} published extensions", published_versions.len());
let known_versions = app_state.db.get_known_extension_versions().await?;
let mut new_versions = HashMap::<&str, Vec<NewExtensionVersion>>::default();
let empty = Vec::new();
for (extension_id, published_versions) in published_versions {
for (extension_id, published_versions) in &published_versions {
let known_versions = known_versions.get(extension_id).unwrap_or(&empty);
for published_version in published_versions {
if known_versions
.binary_search_by_key(&published_version, String::as_str)
.binary_search_by_key(&published_version, |known_version| known_version)
.is_err()
{
if let Some(extension) = fetch_extension_manifest(
blob_store_client,
blob_store_bucket,
extension_id,
published_version,
&extension_id,
&published_version,
)
.await
.log_err()
{
new_versions
.entry(extension_id)
.entry(&extension_id)
.or_default()
.push(extension);
}