script: able to reproduce 4GB bug render.com experienced

scripts/recreate-bucket-bug.js

@@ -1,9 +1,9 @@
 import * as Environment from "~/node_common/environment";
 import * as Utilities from "~/node_common/utilities";
-import * as Social from "~/node_common/social";
 
 import configs from "~/knexfile";
 import knex from "knex";
+import FS from "fs";
 
 import { PrivateKey } from "@textile/hub";
 import { execSync } from "child_process";
@@ -16,12 +16,26 @@ const db = knex(envConfig);
 // We will trigger it by trying to upload a file thats too big.
 console.log(`RUNNING:  recreate-bucket-bug.js`);
 
+const HIGH_WATER_MARK = 1024 * 1024 * 3;
+
 const TEXTILE_KEY_INFO = {
   key: Environment.TEXTILE_HUB_KEY,
   secret: Environment.TEXTILE_HUB_SECRET,
 };
 
+const reportError = (message) => {
+  console.log(
+    `\x1b[1m[ \x1b[31mTEXTILE ERROR\x1b[0m\x1b[1m ]\x1b[0m ${message}`
+  );
+};
+
+const reportTask = (message) => {
+  console.log(`\x1b[1m[ \x1b[32mSCRIPT\x1b[0m\x1b[1m ]\x1b[0m ${message}`);
+};
+
 const run = async () => {
+  reportTask(`creating key`);
+
   const identity = await PrivateKey.fromRandom();
   const api = identity.toString();
 
@@ -34,6 +48,10 @@ const run = async () => {
     },
   };
 
+  reportTask(
+    `initializing bucket with .env variables for your Slate environment`
+  );
+
   console.log({ api });
 
   let { buckets, bucketKey } = await Utilities.getBucketAPIFromUserToken({
@@ -42,19 +60,17 @@ const run = async () => {
   });
 
   // NOTE(jim): Verify this new bucket exists.
+  //
+  // ----------------------------------------------------------------
+
+  reportTask(`attempting to print bucket`);
+
   let list = null;
   try {
     list = await buckets.list();
     console.log({ list });
   } catch (e) {
-    console.log(e.message);
-    Social.sendTextileSlackMessage({
-      file: "recreate-bucket-bug.js",
-      user,
-      message: e.message,
-      code: e.code,
-      functionName: `buckets.list`,
-    });
+    reportError(e.message);
   }
 
   // NOTE(jim): Create 200MB File
@@ -62,55 +78,174 @@ const run = async () => {
   //            Create 1GB File
   //            Create 2GB File
   //            Create 4GB File
+  // ----------------------------------------------------------------
+
+  reportTask(`creating test files, hardcoded in script file`);
+
+  const files = [];
   try {
     await execSync(
       "dd if=/dev/random of=200MB_BUCKET_TEST.txt bs=1 count=0 seek=200m"
     );
+    files.push("200MB_BUCKET_TEST.txt");
+
     await execSync(
       "dd if=/dev/random of=500MB_BUCKET_TEST.txt bs=1 count=0 seek=500m"
     );
+    files.push("500MB_BUCKET_TEST.txt");
+
     await execSync(
       "dd if=/dev/random of=1GB_BUCKET_TEST.txt bs=1 count=0 seek=1g"
     );
+    files.push("1GB_BUCKET_TEST.txt");
+
     await execSync(
       "dd if=/dev/random of=2GB_BUCKET_TEST.txt bs=1 count=0 seek=2g"
     );
+    files.push("2GB_BUCKET_TEST.txt");
+
+    // NOTE(jim): For Textile
+    //            Uncommenting this block will trigger a bug we should
+    //            Resolve first.
+    // ----------------------------------------------------------------
+
+    /*
     await execSync(
       "dd if=/dev/random of=4GB_BUCKET_TEST.txt bs=1 count=0 seek=4g"
     );
+    files.push("4GB_BUCKET_TEST.txt");
+    */
   } catch (e) {
-    console.log(e.message);
+    reportError(e.message);
+  }
+
+  console.log({ files });
+
+  // NOTE(jim): Try to upload each file the first time.
+  // ----------------------------------------------------------------
+  let items;
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+
+    const readStream = FS.createReadStream(`${file}`, {
+      highWaterMark: HIGH_WATER_MARK,
+    });
+
+    reportTask(`attempting ${file} push to bucket`);
+
+    try {
+      await buckets.pushPath(bucketKey, file, readStream);
+    } catch (e) {
+      reportError(e.message);
+    }
+
+    reportTask(`successfully added ${file}`);
+
+    items = null;
+    try {
+      reportTask(`attempting listPath for ${list[0].key}`);
+      const response = await buckets.listPath(list[0].key, "/");
+      items = response.item.items;
+    } catch (e) {
+      reportError(e.message);
+    }
+
+    reportTask(`there are ${items.length} items, including .textileseed`);
+  }
+
+  // NOTE(jim): Remove each file
+  //            We remove each file so that we can upload more files again.
+  // ----------------------------------------------------------------
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+
+    reportTask(`attempting to remove ${item.name}.`);
+    try {
+      await buckets.removePath(bucketKey, item.name);
+      reportTask(`${item.name} has been removed.`);
+    } catch (e) {
+      reportError(e.message);
+    }
+  }
+
+  // NOTE(jim): here we check to make sure that each bucket item
+  //            has been removed appropriately.
+  // ----------------------------------------------------------------
+  try {
+    reportTask(`attempting listPath for ${list[0].key} after deletions`);
+    const response = await buckets.listPath(list[0].key, "/");
+    items = response.item.items;
+  } catch (e) {
+    reportError(e.message);
+  }
+
+  reportTask(`there are ${items.length} items.`);
+
+  // NOTE(jim): Try to upload each file again.
+  //            This helps us check whether or not the bucket can handle
+  //            uploads after files have been removed.
+  // ----------------------------------------------------------------
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+
+    const readStream = FS.createReadStream(`${file}`, {
+      highWaterMark: HIGH_WATER_MARK,
+    });
+
+    reportTask(
+      `\x1b[1m[ second upload phase ]\x1b[0m attempting ${file} push to bucket`
+    );
+
+    try {
+      await buckets.pushPath(bucketKey, file, readStream);
+    } catch (e) {
+      reportError(e.message);
+    }
+
+    reportTask(
+      `\x1b[1m[ second upload phase ]\x1b[0m successfully added ${file}`
+    );
+
+    items = null;
+    try {
+      reportTask(
+        `\x1b[1m[ second upload phase ]\x1b[0m attempting listPath for ${
+          list[0].key
+        }`
+      );
+      const response = await buckets.listPath(list[0].key, "/");
+      items = response.item.items;
+    } catch (e) {
+      reportError(e.message);
+    }
+
+    reportTask(
+      `\x1b[1m[ second upload phase ]\x1b[0m there are ${
+        items.length
+      } items, including .textileseed`
+    );
   }
 
   // NOTE(jim): Remove the bucket from Textile.
+  // ----------------------------------------------------------------
   let bucketRemoval;
   try {
     bucketRemoval = await buckets.remove(list[0].key);
   } catch (e) {
-    console.log(e.message);
-    Social.sendTextileSlackMessage({
-      file: "recreate-bucket-bug.js",
-      user,
-      message: e.message,
-      code: e.code,
-      functionName: `buckets.removePath`,
-    });
+    reportError(e.message);
   }
 
   // NOTE(jim): Verify we cleaned up user data.
+  // ----------------------------------------------------------------
+  list = "CHANGE_ME";
   try {
     list = await buckets.list();
     console.log({ emptyList: list });
   } catch (e) {
-    console.log(e.message);
-    Social.sendTextileSlackMessage({
-      file: "recreate-bucket-bug.js",
-      user,
-      message: e.message,
-      code: e.code,
-      functionName: `buckets.list`,
-    });
+    reportError(e.message);
   }
+
+  reportTask(`SUCCESS.`);
 };
 
 run();