diff --git a/package.json b/package.json index 19f62599..2cd848c2 100644 --- a/package.json +++ b/package.json @@ -38,9 +38,8 @@ "@babel/runtime": "^7.12.5", "@emotion/babel-preset-css-prop": "10.1.0", "@emotion/core": "10.0.35", - "@glif/filecoin-number": "^1.0.0-beta.1", + "@glif/filecoin-number": "^1.0.0-beta.5", "@improbable-eng/grpc-web": "^0.13.0", - "@react-hook/window-size": "^3.0.7", "@slack/webhook": "^5.0.3", "@textile/grpc-transport": "0.0.3", "@textile/hub": "^1.1.1", @@ -55,12 +54,11 @@ "dotenv": "^8.2.0", "express": "^4.17.1", "express-rate-limit": "^5.1.3", - "fs": "0.0.1-security", "fs-extra": "^9.0.1", "heic2any": "0.0.3", "isomorphic-fetch": "^3.0.0", "jsonwebtoken": "^8.5.1", - "knex": "^0.21.5", + "knex": "^0.21.12", "minisearch": "^2.5.1", "moment": "^2.27.0", "morgan": "^1.10.0", diff --git a/scripts/recreate-bucket-bug-render.js b/scripts/recreate-bucket-bug-render.js deleted file mode 100644 index d90f2dde..00000000 --- a/scripts/recreate-bucket-bug-render.js +++ /dev/null @@ -1,265 +0,0 @@ -import * as Environment from "~/node_common/environment"; -import * as Utilities from "~/node_common/utilities"; - -import configs from "~/knexfile"; -import knex from "knex"; -import FS from "fs"; -import AbortController from "abort-controller"; - -import { PrivateKey } from "@textile/hub"; -import { execSync } from "child_process"; - -const envConfig = configs["development"]; -const db = knex(envConfig); - -// NOTE(jim): -// This script will reproduce the bucket bricking bug -// We will trigger it by trying to upload a file thats too big. -console.log(`RUNNING: recreate-bucket-bug-render.js`); - -const HIGH_WATER_MARK = 1024 * 1024 * 3; - -function sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -const TEXTILE_KEY_INFO = { - key: Environment.TEXTILE_HUB_KEY, - secret: Environment.TEXTILE_HUB_SECRET, -}; - -const reportError = (message) => { - console.log( - `\x1b[1m[ \x1b[31mTEXTILE ERROR\x1b[0m\x1b[1m ]\x1b[0m ${message}` - ); -}; - -const reportTask = (message) => { - console.log(`\x1b[1m[ \x1b[32mSCRIPT\x1b[0m\x1b[1m ]\x1b[0m ${message}`); -}; - -const run = async () => { - reportTask(`creating key`); - - const identity = await PrivateKey.fromRandom(); - const api = identity.toString(); - - const user = { - username: "SCRIPT_USER", - data: { - tokens: { - api, - }, - }, - }; - - reportTask( - `initializing bucket with .env variables from your Slate environment` - ); - - console.log({ api }); - - let { - buckets, - bucketKey, - bucketRoot, - } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - // NOTE(jim): Verify this new bucket exists. - // - // ---------------------------------------------------------------- - - reportTask(`attempting to print bucket`); - - let list = null; - try { - list = await buckets.list(); - console.log({ list }); - } catch (e) { - reportError(e.message); - } - - // NOTE(jim): Create 200MB File - // Create 500MB File - // Create 1GB File - // Create 2GB File - // Create 4GB File - // ---------------------------------------------------------------- - - reportTask(`creating test files, hardcoded in script file`); - - const files = []; - try { - await execSync("fallocate -l 200M 200MB_BUCKET_TEST.txt"); - files.push("200MB_BUCKET_TEST.txt"); - - await execSync("fallocate -l 2G 2GB_BUCKET_TEST.txt"); - files.push("2GB_BUCKET_TEST.txt"); - - await execSync("fallocate -l 1G 1GB_BUCKET_TEST.txt"); - files.push("1GB_BUCKET_TEST.txt"); - } catch (e) { - reportError(e.message); - } - - console.log({ files }); - - // NOTE(jim): Try to upload each file the first time. - // ---------------------------------------------------------------- - let firstTry = true; - - let items; - for (let i = 0; i < files.length; i++) { - let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - const file = files[i]; - let controller = new AbortController(); - let { signal } = controller; - const readStream = FS.createReadStream(`${file}`, { - highWaterMark: HIGH_WATER_MARK, - }); - reportTask(`attempting ${file} push to bucket`); - - try { - await buckets.pushPath(bucketKey, file, readStream, { - root: bucketRoot, - signal, - }); - reportTask(`successfully added ${file}`); - } catch (e) { - reportError(e.message); - } - - items = null; - try { - reportTask(`attempting listPath for ${list[0].key}`); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - if (items) { - reportTask(`there are ${items.length} items, including .textileseed`); - } else { - reportError(`listPath failed so there are no items to work with`); - } - } - - // NOTE(jim): Remove each file - // We remove each file so that we can upload more files again. - // ---------------------------------------------------------------- - for (let i = 0; i < items.length; i++) { - const item = items[i]; - - reportTask(`attempting to remove ${item.name}`); - try { - await buckets.removePath(bucketKey, item.name); - reportTask(`${item.name} has been removed`); - } catch (e) { - reportError(e.message); - } - } - - // NOTE(jim): here we check to make sure that each bucket item - // has been removed appropriately. - // ---------------------------------------------------------------- - try { - reportTask(`attempting listPath for ${list[0].key} after deletions`); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - reportTask(`there are ${items.length} items`); - - // NOTE(jim): Try to upload each file again. - // This helps us check whether or not the bucket can handle - // uploads after files have been removed. - // ---------------------------------------------------------------- - for (let i = 0; i < files.length; i++) { - let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - const file = files[i]; - - const readStream = FS.createReadStream(`${file}`, { - highWaterMark: HIGH_WATER_MARK, - }); - - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m attempting ${file} push to bucket` - ); - - try { - await buckets.pushPath(bucketKey, file, readStream); - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m successfully added ${file}` - ); - } catch (e) { - reportError(e.message); - } - - items = null; - try { - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m attempting listPath for ${ - list[0].key - }` - ); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - if (!items) { - reportError( - `\x1b[1m[ second upload phase ]\x1b[0m can't report because listPath failed` - ); - } else { - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m there are ${ - items.length - } items, including .textileseed` - ); - } - } - - // NOTE(jim): Remove the bucket from Textile. - // ---------------------------------------------------------------- - let bucketRemoval; - try { - bucketRemoval = await buckets.remove(list[0].key); - } catch (e) { - reportError(e.message); - } - - // NOTE(jim): Verify we cleaned up user data. - // ---------------------------------------------------------------- - list = "CHANGE_ME"; - try { - list = await buckets.list(); - console.log({ emptyList: list }); - } catch (e) { - reportError(e.message); - } - - reportTask(`SUCCESS.`); -}; - -run(); - -console.log(`FINISHED: recreate-bucket-bug-render.js`); -console.log(` CTRL +C to return to terminal`); diff --git a/scripts/recreate-bucket-bug.js b/scripts/recreate-bucket-bug.js deleted file mode 100644 index f70bfa94..00000000 --- a/scripts/recreate-bucket-bug.js +++ /dev/null @@ -1,307 +0,0 @@ -import * as Environment from "~/node_common/environment"; -import * as Utilities from "~/node_common/utilities"; - -import configs from "~/knexfile"; -import knex from "knex"; -import FS from "fs"; -import AbortController from "abort-controller"; - -import { PrivateKey } from "@textile/hub"; -import { execSync } from "child_process"; - -const envConfig = configs["development"]; -const db = knex(envConfig); - -// NOTE(jim): -// This script will reproduce the bucket bricking bug -// We will trigger it by trying to upload a file thats too big. -console.log(`RUNNING: recreate-bucket-bug.js`); - -const HIGH_WATER_MARK = 1024 * 1024 * 3; - -function sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -const TEXTILE_KEY_INFO = { - key: Environment.TEXTILE_HUB_KEY, - secret: Environment.TEXTILE_HUB_SECRET, -}; - -const reportError = (message) => { - console.log( - `\x1b[1m[ \x1b[31mTEXTILE ERROR\x1b[0m\x1b[1m ]\x1b[0m ${message}` - ); -}; - -const reportTask = (message) => { - console.log(`\x1b[1m[ \x1b[32mSCRIPT\x1b[0m\x1b[1m ]\x1b[0m ${message}`); -}; - -const run = async () => { - reportTask(`creating key`); - - const identity = await PrivateKey.fromRandom(); - const api = identity.toString(); - - const user = { - username: "SCRIPT_USER", - data: { - tokens: { - api, - }, - }, - }; - - reportTask( - `initializing bucket with .env variables from your Slate environment` - ); - - console.log({ api }); - - let { - buckets, - bucketKey, - bucketRoot, - } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - // NOTE(jim): Verify this new bucket exists. - // - // ---------------------------------------------------------------- - - reportTask(`attempting to print bucket`); - - let list = null; - try { - list = await buckets.list(); - console.log({ list }); - } catch (e) { - reportError(e.message); - } - - // NOTE(jim): Create 200MB File - // Create 500MB File - // Create 1GB File - // Create 2GB File - // Create 4GB File - // ---------------------------------------------------------------- - - reportTask(`creating test files, hardcoded in script file`); - - const files = []; - try { - // NOTE(jim): For Textile - // Uncommenting this block will trigger a bug we should - // Resolve first. - // ---------------------------------------------------------------- - - await execSync( - "dd if=/dev/random of=4GB_BUCKET_TEST.txt bs=1 count=0 seek=4g" - ); - files.push("4GB_BUCKET_TEST.txt"); - - await execSync( - "dd if=/dev/random of=200MB_BUCKET_TEST.txt bs=1 count=0 seek=200m" - ); - files.push("200MB_BUCKET_TEST.txt"); - - await execSync( - "dd if=/dev/random of=500MB_BUCKET_TEST.txt bs=1 count=0 seek=500m" - ); - files.push("500MB_BUCKET_TEST.txt"); - - await execSync( - "dd if=/dev/random of=1GB_BUCKET_TEST.txt bs=1 count=0 seek=1g" - ); - files.push("1GB_BUCKET_TEST.txt"); - - await execSync( - "dd if=/dev/random of=2GB_BUCKET_TEST.txt bs=1 count=0 seek=2g" - ); - files.push("2GB_BUCKET_TEST.txt"); - } catch (e) { - reportError(e.message); - } - - console.log({ files }); - - // NOTE(jim): Try to upload each file the first time. - // ---------------------------------------------------------------- - let firstTry = true; - - let items; - for (let i = 0; i < files.length; i++) { - let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - const file = files[i]; - let controller = new AbortController(); - let { signal } = controller; - const readStream = FS.createReadStream(`${file}`, { - highWaterMark: HIGH_WATER_MARK, - }); - reportTask(`attempting ${file} push to bucket`); - - if (firstTry) { - firstTry = false; - - try { - buckets.pushPath(bucketKey, file, readStream, { - root: bucketRoot, - signal, - }); - - await sleep(5000); - - controller.abort(); - - reportTask(`aborted the first file: ${file}`); - - await sleep(5000); - } catch (e) { - reportError(e.message); - } - } else { - try { - await buckets.pushPath(bucketKey, file, readStream, { - root: bucketRoot, - signal, - }); - reportTask(`successfully added ${file}`); - } catch (e) { - reportError(e.message); - } - } - - items = null; - try { - reportTask(`attempting listPath for ${list[0].key}`); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - if (items) { - reportTask(`there are ${items.length} items, including .textileseed`); - } else { - reportError(`listPath failed so there are no items to work with`); - } - } - - // NOTE(jim): Remove each file - // We remove each file so that we can upload more files again. - // ---------------------------------------------------------------- - for (let i = 0; i < items.length; i++) { - const item = items[i]; - - reportTask(`attempting to remove ${item.name}`); - try { - await buckets.removePath(bucketKey, item.name); - reportTask(`${item.name} has been removed`); - } catch (e) { - reportError(e.message); - } - } - - // NOTE(jim): here we check to make sure that each bucket item - // has been removed appropriately. - // ---------------------------------------------------------------- - try { - reportTask(`attempting listPath for ${list[0].key} after deletions`); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - reportTask(`there are ${items.length} items`); - - // NOTE(jim): Try to upload each file again. - // This helps us check whether or not the bucket can handle - // uploads after files have been removed. - // ---------------------------------------------------------------- - for (let i = 0; i < files.length; i++) { - let { bucketRoot } = await Utilities.getBucketAPIFromUserToken({ - user, - bucketName: "delete-this-bucket", - }); - - const file = files[i]; - - const readStream = FS.createReadStream(`${file}`, { - highWaterMark: HIGH_WATER_MARK, - }); - - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m attempting ${file} push to bucket` - ); - - try { - await buckets.pushPath(bucketKey, file, readStream); - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m successfully added ${file}` - ); - } catch (e) { - reportError(e.message); - } - - items = null; - try { - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m attempting listPath for ${ - list[0].key - }` - ); - const response = await buckets.listPath(list[0].key, "/"); - items = response.item.items; - } catch (e) { - reportError(e.message); - } - - if (!items) { - reportError( - `\x1b[1m[ second upload phase ]\x1b[0m can't report because listPath failed` - ); - } else { - reportTask( - `\x1b[1m[ second upload phase ]\x1b[0m there are ${ - items.length - } items, including .textileseed` - ); - } - } - - // NOTE(jim): Remove the bucket from Textile. - // ---------------------------------------------------------------- - let bucketRemoval; - try { - bucketRemoval = await buckets.remove(list[0].key); - } catch (e) { - reportError(e.message); - } - - // NOTE(jim): Verify we cleaned up user data. - // ---------------------------------------------------------------- - list = "CHANGE_ME"; - try { - list = await buckets.list(); - console.log({ emptyList: list }); - } catch (e) { - reportError(e.message); - } - - reportTask(`SUCCESS.`); -}; - -run(); - -console.log(`FINISHED: recreate-bucket-bug.js`); -console.log(` CTRL +C to return to terminal`); diff --git a/scripts/worker-analytics.js b/scripts/worker-analytics.js index 759686d5..6d7bafc8 100644 --- a/scripts/worker-analytics.js +++ b/scripts/worker-analytics.js @@ -1,6 +1,5 @@ import configs from "~/knexfile"; import knex from "knex"; -import fs from "fs"; import * as Data from "~/node_common/data"; import * as Strings from "~/common/strings"; @@ -20,7 +19,7 @@ function sortObject(obj) { }); } } - arr.sort(function (a, b) { + arr.sort(function(a, b) { return b.value - a.value; }); //arr.sort(function(a, b) { a.value.toLowerCase().localeCompare(b.value.toLowerCase()); }); //use this to sort as strings @@ -54,10 +53,6 @@ const run = async () => { console.log("TOTAL USER COUNT", count); console.log("TOTAL BYTES", bytes); console.log("TOTAL BYTES (CONVERTED)", Strings.bytesToSize(bytes)); - - fs.writeFile("analytics.txt", JSON.stringify({ data: userMap }, null, 2), function (err) { - if (err) return console.log(err); - }); }; run();