Merge pull request #365 from filecoin-project/@jimmylee/analytics

Analytics script and storage deal making script.

.gitignore

@@ -11,6 +11,8 @@ package-lock.json
 yarn.lock
 node_modules
 dist
+analytics.txt
+slate-storage-addresses.json
 
 200MB_BUCKET_TEST.txt
 500MB_BUCKET_TEST.txt

common/strings.js

@@ -83,7 +83,7 @@ export const formatAsFilecoinConversion = (number) => {
 };
 
 export const formatAsFilecoin = (number) => {
-  return `${formatNumber(number)} FIL`;
+  return `${number} FIL`;
 };
 
 export const pluralize = (text, count) => {
@@ -171,18 +171,9 @@ export const getRemainingTime = (seconds) => {
 
 export const hexToRGBA = (hex, alpha = 1) => {
   hex = hex.replace("#", "");
-  var r = parseInt(
-    hex.length == 3 ? hex.slice(0, 1).repeat(2) : hex.slice(0, 2),
-    16
-  );
-  var g = parseInt(
-    hex.length == 3 ? hex.slice(1, 2).repeat(2) : hex.slice(2, 4),
-    16
-  );
-  var b = parseInt(
-    hex.length == 3 ? hex.slice(2, 3).repeat(2) : hex.slice(4, 6),
-    16
-  );
+  var r = parseInt(hex.length == 3 ? hex.slice(0, 1).repeat(2) : hex.slice(0, 2), 16);
+  var g = parseInt(hex.length == 3 ? hex.slice(1, 2).repeat(2) : hex.slice(2, 4), 16);
+  var b = parseInt(hex.length == 3 ? hex.slice(2, 3).repeat(2) : hex.slice(4, 6), 16);
   if (alpha) {
     return "rgba(" + r + ", " + g + ", " + b + ", " + alpha + ")";
   } else {
@@ -214,10 +205,7 @@ export const createSlug = (text, base = "untitled") => {
     return base;
   }
 
-  text = text
-    .toString()
-    .toLowerCase()
-    .trim();
+  text = text.toString().toLowerCase().trim();
 
   const sets = [
     { to: "a", from: "[ÀÁÂÃÅÆĀĂĄẠẢẤẦẨẪẬẮẰẲẴẶ]" },

node_common/data/methods/get-every-user.js

@@ -2,7 +2,7 @@ import * as Serializers from "~/node_common/serializers";
 
 import { runQuery } from "~/node_common/data/utilities";
 
-export default async () => {
+export default async (sanitize = true) => {
   return await runQuery({
     label: "GET_EVERY_USER",
     queryFn: async (DB) => {
@@ -12,8 +12,12 @@ export default async () => {
         return [];
       }
 
-      const sanitized = r.map((each) => Serializers.user(each));
-      return JSON.parse(JSON.stringify(sanitized));
+      if (sanitize) {
+        const sanitized = r.map((each) => Serializers.user(each));
+        return JSON.parse(JSON.stringify(sanitized));
+      }
+
+      return JSON.parse(JSON.stringify(r));
     },
     errorFn: async (e) => {
       console.log({

node_common/script-logging.js

@@ -0,0 +1,47 @@
+const { performance } = require("perf_hooks");
+
+const TIME_START = performance.now();
+
+const timeDifference = (current, previous) => {
+  var msPerMinute = 60 * 1000;
+  var msPerHour = msPerMinute * 60;
+  var msPerDay = msPerHour * 24;
+  var msPerMonth = msPerDay * 30;
+  var msPerYear = msPerDay * 365;
+
+  var elapsed = current - previous;
+
+  if (elapsed < msPerMinute) {
+    return Math.round(elapsed / 1000) + " seconds ago";
+  } else if (elapsed < msPerHour) {
+    return Math.round(elapsed / msPerMinute) + " minutes ago";
+  } else if (elapsed < msPerDay) {
+    return Math.round(elapsed / msPerHour) + " hours ago";
+  } else if (elapsed < msPerMonth) {
+    return "approximately " + Math.round(elapsed / msPerDay) + " days ago";
+  } else if (elapsed < msPerYear) {
+    return "approximately " + Math.round(elapsed / msPerMonth) + " months ago";
+  } else {
+    return "approximately " + Math.round(elapsed / msPerYear) + " years ago";
+  }
+};
+
+const setTime = () => {
+  return `[ \x1b[33m\x1b[5m${timeDifference(performance.now(), TIME_START)}\x1b[0m ]`;
+};
+
+export const error = (message, name = "ERROR ") => {
+  console.log(`\x1b[1m[ \x1b[31m${name}\x1b[0m\x1b[1m ]\x1b[0m ${setTime()} ${message}`);
+};
+
+export const task = (message, name = "SCRIPT") => {
+  console.log(`\x1b[1m[ \x1b[32m${name}\x1b[0m\x1b[1m ]\x1b[0m ${setTime()} ${message}`);
+};
+
+export const taskTimeless = (message, name = "SCRIPT") => {
+  console.log(`\x1b[1m[ \x1b[32m${name}\x1b[0m\x1b[1m ]\x1b[0m ${message}`);
+};
+
+export const note = (message, name = "NOOP  ") => {
+  console.log(`\x1b[1m[ \x1b[33m${name}\x1b[0m\x1b[1m ]\x1b[0m ${setTime()} ${message}`);
+};

node_common/utilities.js

@@ -144,8 +144,6 @@ export const setupWithThread = async ({ buckets }) => {
     const res = await client.getThread("buckets");
 
     buckets.withThread(res.id.toString());
-
-    console.log(`[ buckets ] getThread success`);
   } catch (error) {
     if (error.message !== "Thread not found") {
       throw new Error(error.message);
@@ -156,19 +154,13 @@ export const setupWithThread = async ({ buckets }) => {
     const threadID = newId.toString();
 
     buckets.withThread(threadID);
-
-    console.log(`[ buckets ] newDB success`);
   }
 
   return buckets;
 };
 
 // NOTE(jim): Requires @textile/hub
-export const getBucketAPIFromUserToken = async ({
-  user,
-  bucketName,
-  encrypted = false,
-}) => {
+export const getBucketAPIFromUserToken = async ({ user, bucketName, encrypted = false }) => {
   const token = user.data.tokens.api;
   const name = Strings.isEmpty(bucketName) ? BUCKET_NAME : bucketName;
   const identity = await PrivateKey.fromString(token);

package.json

@@ -17,9 +17,7 @@
     "electron-prod": "NODE_ENV=production next build && electron-builder --dir",
     "scripts": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts",
     "www-setup-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts setup-database",
-    "www-seed-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts seed-database",
-    "www-drop-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts drop-database",
-    "www-adjust-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts adjust"
+    "www-seed-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts seed-database"
   },
   "build": {
     "appId": "com.slate",

scripts/worker-analytics.js

@@ -0,0 +1,66 @@
+import configs from "~/knexfile";
+import knex from "knex";
+import fs from "fs";
+
+import * as Data from "~/node_common/data";
+import * as Strings from "~/common/strings";
+
+const envConfig = configs["development"];
+const db = knex(envConfig);
+
+console.log(`RUNNING: worker-analytics.js`);
+
+function sortObject(obj) {
+  var arr = [];
+  for (var prop in obj) {
+    if (obj.hasOwnProperty(prop)) {
+      arr.push({
+        key: prop,
+        value: obj[prop],
+      });
+    }
+  }
+  arr.sort(function (a, b) {
+    return b.value - a.value;
+  });
+  //arr.sort(function(a, b) { a.value.toLowerCase().localeCompare(b.value.toLowerCase()); }); //use this to sort as strings
+  return arr; // returns array
+}
+
+const run = async () => {
+  const response = await Data.getEveryUser(false);
+
+  let count = 0;
+  let bytes = 0;
+  let userMap = {};
+
+  response.forEach((user) => {
+    count = count + 1;
+
+    let userBytes = 0;
+    user.data.library[0].children.forEach((each) => {
+      userBytes = each.size + userBytes;
+      bytes = each.size + bytes;
+    });
+
+    userMap[user.username] = userBytes;
+  });
+
+  userMap = sortObject(userMap);
+  userMap = userMap.map((each, index) => {
+    return { ...each, index, value: Strings.bytesToSize(each.value) };
+  });
+  console.log(userMap);
+  console.log("TOTAL USER COUNT", count);
+  console.log("TOTAL BYTES", bytes);
+  console.log("TOTAL BYTES (CONVERTED)", Strings.bytesToSize(bytes));
+
+  fs.writeFile("analytics.txt", JSON.stringify({ data: userMap }, null, 2), function (err) {
+    if (err) return console.log(err);
+  });
+};
+
+run();
+
+console.log(`FINISHED: worker-analytics.js`);
+console.log(`          CTRL +C to return to terminal.`);

scripts/worker-heavy-stones.js

@@ -0,0 +1,311 @@
+import fs from "fs-extra";
+
+import * as Environment from "~/node_common/environment";
+import * as Data from "~/node_common/data";
+import * as Utilities from "~/node_common/utilities";
+import * as Strings from "~/common/strings";
+import * as Logs from "~/node_common/script-logging";
+
+import { Buckets, PrivateKey } from "@textile/hub";
+import { v4 as uuid } from "uuid";
+
+// 50 MB minimum
+const MINIMUM_BYTES_FOR_STORAGE = 52428800;
+const STORAGE_BOT_NAME = "STORAGE WORKER";
+const PRACTICE_RUN = false;
+const SKIP_NEW_BUCKET_CREATION = true;
+
+const TEXTILE_KEY_INFO = {
+  key: Environment.TEXTILE_HUB_KEY,
+  secret: Environment.TEXTILE_HUB_SECRET,
+};
+
+console.log(`RUNNING:  worker-heavy-stones.js`);
+
+const run = async () => {
+  const response = await Data.getEveryUser(false);
+
+  const storageUsers = [];
+  const writable = [];
+  const slateAddresses = [];
+  let bytes = 0;
+  let dealUsers = 0;
+  let totalUsers = 0;
+  let encryptedUsers = 0;
+
+  // NOTE(jim): Only users who agree. Opt in by default.
+  for (let i = 0; i < response.length; i++) {
+    const user = response[i];
+
+    if (user.data.allow_automatic_data_storage) {
+      storageUsers.push(user);
+      dealUsers = dealUsers + 1;
+    }
+
+    if (user.data.allow_encrypted_data_storage) {
+      encryptedUsers = encryptedUsers + 1;
+    }
+
+    totalUsers = totalUsers + 1;
+  }
+
+  for (let i = 0; i < storageUsers.length; i++) {
+    const user = storageUsers[i];
+    const printData = {
+      username: storageUsers[i].username,
+      slateURL: `https://slate.host/${storageUsers[i].username}`,
+      isForcingEncryption: user.data.allow_encrypted_data_storage,
+    };
+    let buckets;
+
+    try {
+      const token = user.data.tokens.api;
+      const identity = await PrivateKey.fromString(token);
+      buckets = await Buckets.withKeyInfo(TEXTILE_KEY_INFO);
+      await buckets.getToken(identity);
+      buckets = await Utilities.setupWithThread({ buckets });
+    } catch (e) {
+      Logs.error(e.message);
+    }
+
+    let userBuckets = [];
+    try {
+      userBuckets = await buckets.list();
+    } catch (e) {
+      Logs.error(e.message);
+    }
+
+    let userBytes = 0;
+
+    for (let k = 0; k < userBuckets.length; k++) {
+      try {
+        const path = await buckets.listPath(userBuckets[k].key, "/");
+        const data = path.item;
+
+        if (data.name !== "data") {
+          continue;
+        }
+
+        userBuckets[k].bucketSize = data.size;
+        userBytes = userBytes + data.size;
+        bytes = bytes + userBytes;
+      } catch (e) {
+        Logs.error(e.message);
+      }
+    }
+
+    printData.bytes = userBytes;
+
+    // NOTE(jim): Skip people.
+    if (userBytes < MINIMUM_BYTES_FOR_STORAGE) {
+      Logs.note(`SKIP: ${user.username}`);
+      continue;
+    }
+
+    const PowergateSingleton = await Utilities.getPowergateAPIFromUserToken({
+      user,
+    });
+    const { powerInfo, power } = PowergateSingleton;
+    let balance = 0;
+    let address = null;
+
+    try {
+      if (powerInfo) {
+        powerInfo.balancesList.forEach((a) => {
+          balance = a.balance;
+          address = a.addr.addr;
+        });
+      } else {
+        Logs.error(`Powergate powerInfo does not exist.`);
+      }
+    } catch (e) {
+      Logs.error(e.message);
+    }
+
+    let storageDeals = [];
+    try {
+      const listStorageResult = await power.listStorageDealRecords({
+        ascending: false,
+        includePending: false,
+        includeFinal: true,
+      });
+
+      listStorageResult.recordsList.forEach((o) => {
+        storageDeals.push({
+          dealId: o.dealInfo.dealId,
+          rootCid: o.rootCid,
+          proposalCid: o.dealInfo.proposalCid,
+          pieceCid: o.dealInfo.pieceCid,
+          addr: o.addr,
+          miner: o.dealInfo.miner,
+          size: o.dealInfo.size,
+          // NOTE(jim): formatted size.
+          formattedSize: Strings.bytesToSize(o.dealInfo.size),
+          pricePerEpoch: o.dealInfo.pricePerEpoch,
+          startEpoch: o.dealInfo.startEpoch,
+          // NOTE(jim): just for point of reference on the total cost.
+          totalSpeculatedCost: Strings.formatAsFilecoinConversion(
+            o.dealInfo.pricePerEpoch * o.dealInfo.duration
+          ),
+          duration: o.dealInfo.duration,
+          activationEpoch: o.dealInfo.activationEpoch,
+          time: o.time,
+          pending: o.pending,
+        });
+      });
+    } catch (e) {
+      Logs.error(e.message);
+    }
+
+    printData.address = address;
+    printData.balanceAttoFil = balance;
+
+    Logs.taskTimeless(`\x1b[36m\x1b[1mhttps://slate.host/${user.username}\x1b[0m`);
+    Logs.taskTimeless(`\x1b[36m\x1b[1m${address}\x1b[0m`);
+    Logs.taskTimeless(`\x1b[36m\x1b[1m${Strings.bytesToSize(userBytes)} stored each deal.\x1b[0m`);
+    Logs.taskTimeless(
+      `\x1b[36m\x1b[1m${Strings.formatAsFilecoinConversion(balance)} remaining\x1b[0m`
+    );
+
+    console.log(storageDeals);
+
+    // NOTE(jim): Skip users that are out of funds.
+    if (balance === 0) {
+      Logs.error(`OUT OF FUNDS: ${user.username}`);
+      continue;
+    }
+
+    if (address) {
+      slateAddresses.push(address);
+    }
+
+    // NOTE(jim): tracks all buckets.
+    printData.buckets = [];
+
+    for (let j = 0; j < userBuckets.length; j++) {
+      const keyBucket = userBuckets[j];
+      let key;
+      let encrypt;
+
+      if (keyBucket.name.startsWith("open-")) {
+        Logs.note(`bucket found: open-data ${keyBucket.key}`);
+        key = keyBucket.key;
+      }
+
+      if (keyBucket.name.startsWith("encrypted-data-")) {
+        Logs.note(`bucket found: encrypted-data ${keyBucket.key}`);
+        key = keyBucket.key;
+      }
+
+      if (keyBucket.name === "data" && !SKIP_NEW_BUCKET_CREATION) {
+        key = null;
+        encrypt = !!user.data.allow_encrypted_data_storage;
+
+        // NOTE(jim): Create a new bucket
+        const newBucketName = encrypt ? `encrypted-data-${uuid()}` : `open-data-${uuid()}`;
+
+        // NOTE(jim): Get the root key of the bucket
+        let items;
+        try {
+          const path = await buckets.listPath(keyBucket.key, "/");
+          items = path.item;
+        } catch (e) {
+          Logs.error(e.message);
+        }
+
+        Logs.task(`creating new bucket: ${newBucketName}.`);
+
+        // NOTE(jim): Create a new bucket
+        try {
+          Logs.note(`attempting ...`);
+
+          if (!PRACTICE_RUN) {
+            let newBucket = await buckets.create(newBucketName, encrypt, items.cid);
+
+            key = newBucket.root.key;
+
+            Logs.task(`created ${newBucketName} successfully.`);
+          } else {
+            Logs.note(`practice skipping ...`);
+          }
+        } catch (e) {
+          Logs.error(e.message);
+        }
+      }
+
+      if (key) {
+        try {
+          if (!PRACTICE_RUN) {
+            await buckets.archive(key);
+            Logs.task(`\x1b[32mNEW DEAL SUCCESSFUL !!!\x1b[0m`);
+          } else {
+            Logs.note(`archive skipping ...`);
+          }
+
+          printData.buckets.push({
+            key,
+            success: false,
+          });
+        } catch (e) {
+          if (e.message === `the same bucket cid is already archived successfully`) {
+            printData.buckets.push({
+              key,
+              success: true,
+            });
+          } else {
+            printData.buckets.push({
+              key,
+              success: false,
+            });
+          }
+
+          Logs.note(e.message);
+        }
+      }
+    }
+
+    writable.push(printData);
+
+    for (let k = 0; k < printData.buckets.length; k++) {
+      let targetBucket = printData.buckets[k];
+
+      if (targetBucket.success) {
+        try {
+          Logs.task(`deleting bucket with key: ${targetBucket.key}`);
+          await buckets.remove(targetBucket.key);
+          Logs.task(`successfully deleted ${targetBucket.key}`);
+        } catch (e) {
+          Logs.error(e.message);
+        }
+      }
+    }
+
+    console.log("\n");
+  }
+
+  Logs.task(`total storage per run: ${Strings.bytesToSize(bytes)}`);
+  Logs.task(`total storage per run (with replication x5): ${Strings.bytesToSize(bytes * 5)}`);
+  Logs.task(`creating slate-storage-addresses.json`);
+
+  fs.writeFile(
+    "slate-storage-addresses.json",
+    JSON.stringify(
+      {
+        rootAddress:
+          "t3xhj6odc2cjj3z6kmxqugjjai2unacme65gnwigse4xx6jcpmfmi6jg6miqintibacluxi4ydlmolfpruznba",
+        addresses: slateAddresses,
+      },
+      null,
+      2
+    ),
+    function (e) {
+      if (e) return Logs.error(e.message);
+    }
+  );
+
+  console.log(`${consoleName(STORAGE_BOT_NAME)} finished. \n\n`);
+  console.log(`FINISHED: worker-heavy-stones.js`);
+  console.log(`          CTRL +C to return to terminal.`);
+};
+
+run();