diff --git a/package.json b/package.json index dd005a5e..ff49170e 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "www-seed-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts seed-database", "www-adjust-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts adjust", "www-migrate-database": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts flattening-migration", + "www-textile-migration": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts textile-migration-to-estuary", "www-storage-deals": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts worker-storage-deals", "www-search": "NODE_TLS_REJECT_UNAUTHORIZED=0 node ./scripts search" }, diff --git a/scripts/textile-migration-to-estuary.js b/scripts/textile-migration-to-estuary.js new file mode 100644 index 00000000..72c7285a --- /dev/null +++ b/scripts/textile-migration-to-estuary.js @@ -0,0 +1,78 @@ +import "isomorphic-fetch"; + +import * as Environment from "~/node_common/environment"; +import * as Logging from "~/common/logging"; + +import configs from "~/knexfile"; +import knex from "knex"; +import fs from "fs"; + +const ERROR_FILE = "messages.txt"; + +const envConfig = configs["development"]; + +const DB = knex(envConfig); + +// TODOs (migrate all bucketCIDs to estuary) -> (automatically migrate all files) +// Retrieve all bucketCIDs +// upload them one by one to estuary +// Track errors + +async function retrieveBucketsCIDs() { + const bucketsCIDs = await DB.select("textileBucketCID") + .from("users") + .whereNotNull("textileBucketCID"); + + return bucketsCIDs; +} + +async function pinToEstuary(cid) { + let res = await fetch("https://api.estuary.tech/content/add-ipfs", { + method: "POST", + headers: { + Authorization: `Bearer ${Environment.ESTUARY_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + cid, + }), + }); + let json = await res.json(); + return json; +} + +async function runMigration() { + Logging.log(`Migration started`); + const failedBuckets = []; + + const bucketsCIDs = await retrieveBucketsCIDs(); + for (let { textileBucketCID } of bucketsCIDs) { + try { + const response = await pinToEstuary(textileBucketCID); + + if (response?.error) { + Logging.error({ + error: response.error, + decorator: "ADD_CID_TO_ESTUARY", + }); + + fs.appendFileSync(ERROR_FILE, `${textileBucketCID},${JSON.stringify(response.error)}\n`); + failedBuckets.push(textileBucketCID); + } else { + Logging.log(`Bucket ${textileBucketCID} has been uploaded`, response); + } + } catch (e) { + fs.appendFileSync(ERROR_FILE, `${textileBucketCID}, ${JSON.stringify(e)}\n`); + failedBuckets.push(textileBucketCID); + Logging.error({ + error: e, + decorator: "ADD_CID_TO_ESTUARY", + }); + } + } + + fs.appendFileSync(ERROR_FILE, JSON.stringify(failedBuckets)); + console.log("SCRIPT FINISHED", failedBuckets); +} + +runMigration();