From 38c89df330947215e7500f604b2b5c8bd199c791 Mon Sep 17 00:00:00 2001
From: Yury Semikhatsky <yurys@chromium.org>
Date: Tue, 30 May 2023 16:46:37 -0700
Subject: [PATCH] chore: replace "blob download-batch" with a bash script
 (#23381)

`az storage blob download-batch` has been timing out over the last few
days, see upstream issue
https://github.com/Azure/azure-cli/issues/26567. Replacing it with a
simple bash script that discovers blobs with a given prefix and then
downloads one-by-one.
---
 .github/workflows/tests_primary.yml   | 16 ++++++++++++++--
 .github/workflows/tests_secondary.yml |  8 +++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml
index 9e3881dd0d..2f2a127dc3 100644
--- a/.github/workflows/tests_primary.yml
+++ b/.github/workflows/tests_primary.yml
@@ -84,7 +84,13 @@ jobs:
     - run: npm run build
     - name: Download Blob Reports from Azure Blob Storage
       run: |
-        az storage blob download-batch -d . -s '$web'  --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_linux/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        run_dir='run-${{ github.run_id }}-${{ github.sha }}-test_linux'
+        mkdir -p $run_dir
+        LIST=$(az storage blob list -c '$web' --prefix $run_dir --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}")
+        for name in $(echo $LIST | jq --raw-output '.[].name | select(test(".jsonl$"))');
+        do
+          az storage blob download -c '$web' --name $name -f $name  --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        done
     - name: Merge into HTML Report
       run: |
         npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_linux'
@@ -207,7 +213,13 @@ jobs:
     - run: npm run build
     - name: Download Blob Reports from Azure Blob Storage
       run: |
-        az storage blob download-batch -d . -s '$web'  --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        run_dir='run-${{ github.run_id }}-${{ github.sha }}-test_test_runner'
+        mkdir -p $run_dir
+        LIST=$(az storage blob list -c '$web' --prefix $run_dir --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}")
+        for name in $(echo $LIST | jq --raw-output '.[].name | select(test(".jsonl$"))');
+        do
+          az storage blob download -c '$web' --name $name -f $name  --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        done
     - name: Merge into HTML Report
       run: |
         npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner'
diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml
index 956ea6a9a8..a59737abe9 100644
--- a/.github/workflows/tests_secondary.yml
+++ b/.github/workflows/tests_secondary.yml
@@ -231,7 +231,13 @@ jobs:
     - run: npm run build
     - name: Download Blob Reports from Azure Blob Storage
       run: |
-        az storage blob download-batch -d . -s '$web'  --pattern 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        run_dir='run-${{ github.run_id }}-${{ github.sha }}-tracing_linux'
+        mkdir -p $run_dir
+        LIST=$(az storage blob list -c '$web' --prefix $run_dir --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}")
+        for name in $(echo $LIST | jq --raw-output '.[].name | select(test(".jsonl$"))');
+        do
+          az storage blob download -c '$web' --name $name -f $name  --connection-string  "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        done
     - name: Merge into HTML Report
       run: |
         npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux'