slate/scripts/recreate-bucket-bug.js

import * as Environment from "~/node_common/environment";
import * as Utilities from "~/node_common/utilities";

import configs from "~/knexfile";
import knex from "knex";
import FS from "fs";
import AbortController from "abort-controller";

import { PrivateKey } from "@textile/hub";
import { execSync } from "child_process";

const envConfig = configs["development"];
const db = knex(envConfig);

// NOTE(jim):
// This script will reproduce the bucket bricking bug
// We will trigger it by trying to upload a file thats too big.
console.log(`RUNNING:  recreate-bucket-bug.js`);

const HIGH_WATER_MARK = 1024 * 1024 * 3;

function sleep(ms) {
  return new Promise((resolve) => {
    setTimeout(resolve, ms);
  });
}

const TEXTILE_KEY_INFO = {
  key: Environment.TEXTILE_HUB_KEY,
  secret: Environment.TEXTILE_HUB_SECRET,
};

const reportError = (message) => {
  console.log(
    `\x1b[1m[ \x1b[31mTEXTILE ERROR\x1b[0m\x1b[1m ]\x1b[0m ${message}`
  );
};

const reportTask = (message) => {
  console.log(`\x1b[1m[ \x1b[32mSCRIPT\x1b[0m\x1b[1m ]\x1b[0m ${message}`);
};

const run = async () => {
  reportTask(`creating key`);

  const identity = await PrivateKey.fromRandom();
  const api = identity.toString();

  const user = {
    username: "SCRIPT_USER",
    data: {
      tokens: {
        api,
      },
    },
  };

  reportTask(
    `initializing bucket with .env variables from your Slate environment`
  );

  console.log({ api });

  let {
    buckets,
    bucketKey,
    bucketRoot,
  } = await Utilities.getBucketAPIFromUserToken({
    user,
    bucketName: "delete-this-bucket",
  });

  // NOTE(jim): Verify this new bucket exists.
  //
  // ----------------------------------------------------------------

  reportTask(`attempting to print bucket`);

  let list = null;
  try {
    list = await buckets.list();
    console.log({ list });
  } catch (e) {
    reportError(e.message);
  }

  // NOTE(jim): Create 200MB File
  //            Create 500MB File
  //            Create 1GB File
  //            Create 2GB File
  //            Create 4GB File
  // ----------------------------------------------------------------

  reportTask(`creating test files, hardcoded in script file`);

  const files = [];
  try {
    // NOTE(jim): For Textile
    //            Uncommenting this block will trigger a bug we should
    //            Resolve first.
    // ----------------------------------------------------------------

    await execSync(
      "dd if=/dev/random of=4GB_BUCKET_TEST.txt bs=1 count=0 seek=4g"
    );
    files.push("4GB_BUCKET_TEST.txt");

    await execSync(
      "dd if=/dev/random of=200MB_BUCKET_TEST.txt bs=1 count=0 seek=200m"
    );
    files.push("200MB_BUCKET_TEST.txt");

    await execSync(
      "dd if=/dev/random of=500MB_BUCKET_TEST.txt bs=1 count=0 seek=500m"
    );
    files.push("500MB_BUCKET_TEST.txt");

    await execSync(
      "dd if=/dev/random of=1GB_BUCKET_TEST.txt bs=1 count=0 seek=1g"
    );
    files.push("1GB_BUCKET_TEST.txt");

    await execSync(
      "dd if=/dev/random of=2GB_BUCKET_TEST.txt bs=1 count=0 seek=2g"
    );
    files.push("2GB_BUCKET_TEST.txt");
  } catch (e) {
    reportError(e.message);
  }

  console.log({ files });

  // NOTE(jim): Try to upload each file the first time.
  // ----------------------------------------------------------------
  let firstTry = true;

  let items;
  for (let i = 0; i < files.length; i++) {
    let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({
      user,
      bucketName: "delete-this-bucket",
    });

    const file = files[i];
    let controller = new AbortController();
    let { signal } = controller;
    const readStream = FS.createReadStream(`${file}`, {
      highWaterMark: HIGH_WATER_MARK,
    });
    reportTask(`attempting ${file} push to bucket`);

    if (firstTry) {
      firstTry = false;

      try {
        buckets.pushPath(bucketKey, file, readStream, {
          root: bucketRoot,
          signal,
        });

        await sleep(5000);

        controller.abort();

        reportTask(`aborted the first file: ${file}`);

        await sleep(5000);
      } catch (e) {
        reportError(e.message);
      }
    } else {
      try {
        await buckets.pushPath(bucketKey, file, readStream, {
          root: bucketRoot,
          signal,
        });
        reportTask(`successfully added ${file}`);
      } catch (e) {
        reportError(e.message);
      }
    }

    items = null;
    try {
      reportTask(`attempting listPath for ${list[0].key}`);
      const response = await buckets.listPath(list[0].key, "/");
      items = response.item.items;
    } catch (e) {
      reportError(e.message);
    }

    if (items) {
      reportTask(`there are ${items.length} items, including .textileseed`);
    } else {
      reportError(`listPath failed so there are no items to work with`);
    }
  }

  // NOTE(jim): Remove each file
  //            We remove each file so that we can upload more files again.
  // ----------------------------------------------------------------
  for (let i = 0; i < items.length; i++) {
    const item = items[i];

    reportTask(`attempting to remove ${item.name}`);
    try {
      await buckets.removePath(bucketKey, item.name);
      reportTask(`${item.name} has been removed`);
    } catch (e) {
      reportError(e.message);
    }
  }

  // NOTE(jim): here we check to make sure that each bucket item
  //            has been removed appropriately.
  // ----------------------------------------------------------------
  try {
    reportTask(`attempting listPath for ${list[0].key} after deletions`);
    const response = await buckets.listPath(list[0].key, "/");
    items = response.item.items;
  } catch (e) {
    reportError(e.message);
  }

  reportTask(`there are ${items.length} items`);

  // NOTE(jim): Try to upload each file again.
  //            This helps us check whether or not the bucket can handle
  //            uploads after files have been removed.
  // ----------------------------------------------------------------
  for (let i = 0; i < files.length; i++) {
    let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({
      user,
      bucketName: "delete-this-bucket",
    });

    const file = files[i];

    const readStream = FS.createReadStream(`${file}`, {
      highWaterMark: HIGH_WATER_MARK,
    });

    reportTask(
      `\x1b[1m[ second upload phase ]\x1b[0m attempting ${file} push to bucket`
    );

    try {
      await buckets.pushPath(bucketKey, file, readStream);
      reportTask(
        `\x1b[1m[ second upload phase ]\x1b[0m successfully added ${file}`
      );
    } catch (e) {
      reportError(e.message);
    }

    items = null;
    try {
      reportTask(
        `\x1b[1m[ second upload phase ]\x1b[0m attempting listPath for ${
          list[0].key
        }`
      );
      const response = await buckets.listPath(list[0].key, "/");
      items = response.item.items;
    } catch (e) {
      reportError(e.message);
    }

    if (!items) {
      reportError(
        `\x1b[1m[ second upload phase ]\x1b[0m can't report because listPath failed`
      );
    } else {
      reportTask(
        `\x1b[1m[ second upload phase ]\x1b[0m there are ${
          items.length
        } items, including .textileseed`
      );
    }
  }

  // NOTE(jim): Remove the bucket from Textile.
  // ----------------------------------------------------------------
  let bucketRemoval;
  try {
    bucketRemoval = await buckets.remove(list[0].key);
  } catch (e) {
    reportError(e.message);
  }

  // NOTE(jim): Verify we cleaned up user data.
  // ----------------------------------------------------------------
  list = "CHANGE_ME";
  try {
    list = await buckets.list();
    console.log({ emptyList: list });
  } catch (e) {
    reportError(e.message);
  }

  reportTask(`SUCCESS.`);
};

run();

console.log(`FINISHED: recreate-bucket-bug.js`);
console.log(`          CTRL +C to return to terminal`);