slate/scripts/flattening-migration.js

451 lines
14 KiB
JavaScript
Raw Normal View History

2021-08-29 01:15:35 +03:00
import configs from "~/knexfile";
import knex from "knex";
import { v4 as uuid } from "uuid";
import * as Logging from "~/common/logging";
import * as Utilities from "~/node_common/utilities";
import * as Data from "~/node_common/data";
import * as Strings from "~/common/strings";
import * as Constants from "~/common/constants";
const envConfig = configs["development"];
const DB = knex(envConfig);
Logging.log(`RUNNING: files-migration.js`);
/* Check which properties exist */
const printUsersTable = async () => {
const users = await DB.select("*").from("users");
let dataParams = {};
let twitterParams = {};
let statusParams = {};
let onboardingParams = {};
let settingsParams = {};
let tokensParams = {};
for (let user of users) {
for (let param of Object.keys(user.data)) {
if (!dataParams[param]) {
dataParams[param] = true;
}
}
if (user.data.twitter) {
for (let param of Object.keys(user.data.twitter)) {
if (!twitterParams[param]) {
twitterParams[param] = true;
}
}
}
if (user.data.status) {
for (let param of Object.keys(user.data.status)) {
if (!statusParams[param]) {
statusParams[param] = true;
}
}
}
if (user.data.onboarding) {
for (let param of Object.keys(user.data.onboarding)) {
if (!onboardingParams[param]) {
onboardingParams[param] = true;
}
}
}
if (user.data.settings) {
for (let param of Object.keys(user.data.settings)) {
if (!settingsParams[param]) {
settingsParams[param] = true;
}
}
}
if (user.data.tokens) {
for (let param of Object.keys(user.data.tokens)) {
if (!tokensParams[param]) {
tokensParams[param] = true;
}
}
}
}
Logging.log({ dataParams: Object.keys(dataParams) });
Logging.log({ twitterParams: Object.keys(twitterParams) });
Logging.log({ statusParams: Object.keys(statusParams) });
Logging.log({ onboardingParams: Object.keys(onboardingParams) });
Logging.log({ settingsParams: Object.keys(settingsParams) });
Logging.log({ tokensParams: Object.keys(tokensParams) });
};
const printSlatesTable = async () => {
const slates = await DB.select("*").from("slates");
let dataParams = {};
for (let slate of slates) {
for (let param of Object.keys(slate.data)) {
if (!dataParams[param]) {
dataParams[param] = true;
}
}
}
Logging.log({ dataParams: Object.keys(dataParams) });
};
const printFilesTable = async () => {
const files = await DB.select("*").from("files");
let dataParams = {};
let linkParams = {};
for (let file of files) {
for (let param of Object.keys(file.data)) {
if (!dataParams[param]) {
dataParams[param] = true;
}
}
if (file.data.link) {
for (let param of Object.keys(file.data.link)) {
if (!linkParams[param]) {
linkParams[param] = true;
}
}
}
}
Logging.log({ dataParams: Object.keys(dataParams) });
Logging.log({ linkParams: Object.keys(linkParams) });
};
/* Add columns (including tags) */
const createSurveysTable = async () => {
await DB.schema.createTable("surveys", function (table) {
table.uuid("id").primary().unique().notNullable().defaultTo(DB.raw("uuid_generate_v4()"));
table.uuid("ownerId").references("id").inTable("users");
// What do you currently use for saving things on the web?
table.boolean("prevToolsBrowserBookmarks").defaultTo(false);
table.boolean("prevToolsPinterest").defaultTo(false);
table.boolean("prevToolsArena").defaultTo(false);
table.boolean("prevToolsNotesPlatform").defaultTo(false);
table.string("prevToolsOther").defaultTo(null);
// What are you interested in using Slate for?
table.boolean("useCasesBookmarkingImportantPages").defaultTo(false);
table.boolean("useCasesSavingLinksToReadLater").defaultTo(false);
table.boolean("useCasesSearchingYourBrowsedPages").defaultTo(false);
table.boolean("useCasesSharingCollectionsOfLinks").defaultTo(false);
table.string("useCasesOther").defaultTo(null);
// How did you find out about Slate?
table.boolean("referralFriend").defaultTo(false);
table.boolean("referralTwitter").defaultTo(false);
table.boolean("referralIpfsFilecoinCommunity").defaultTo(false);
table.string("referralOther").defaultTo(null);
});
};
const addUserTextileColumns = async () => {
2021-09-28 07:16:26 +03:00
await DB.schema.table("users", function (table) {
table.string("textileKey").nullable();
table.string("textileToken", 400).nullable();
table.string("textileThreadID").nullable();
table.string("textileBucketCID").nullable();
});
};
2021-09-28 07:16:26 +03:00
2021-08-29 01:15:35 +03:00
const addUserColumns = async () => {
await DB.schema.table("users", function (table) {
table.string("body", 2000).nullable();
2021-08-29 01:15:35 +03:00
table.string("photo").nullable();
table.string("name").nullable();
table.string("twitterUsername").nullable();
table.boolean("twitterVerified").notNullable().defaultTo(false);
table.boolean("hasCompletedSurvey").defaultTo(false);
table.boolean("hasCompletedUploadOnboarding").defaultTo(false);
table.boolean("hasCompletedSlatesOnboarding").defaultTo(false);
table.boolean("isFilterSidebarCollapsed").defaultTo(false);
2021-08-29 01:15:35 +03:00
});
};
const addSlateColumns = async () => {
await DB.schema.table("slates", function (table) {
table.string("body", 2000).nullable();
2021-08-29 01:15:35 +03:00
table.string("name").nullable();
table.jsonb("coverImage").nullable();
2021-08-29 01:15:35 +03:00
});
};
//apply this to all other databases as well
const alterFileColumns = async () => {
await DB.schema.alterTable("files", function (table) {
table.string("linkImage", 2000).nullable().alter();
table.string("url", 2000).nullable().alter();
table.string("linkFavicon", 2000).nullable().alter();
});
};
2021-08-29 01:15:35 +03:00
const addFileColumns = async () => {
await DB.schema.table("files", function (table) {
table.renameColumn("data", "oldData");
});
await DB.schema.table("files", function (table) {
table.string("name").nullable();
table.integer("size").notNullable().defaultTo(0);
table.string("type").notNullable().defaultTo("link");
table.string("blurhash").nullable();
table.string("body", 2000).nullable();
2021-08-29 01:15:35 +03:00
table.jsonb("coverImage").nullable();
table.jsonb("data").nullable(); //where you'll move unity stuff
table.string("linkName").nullable();
table.string("linkBody", 2000).nullable();
table.string("linkAuthor").nullable();
table.string("linkSource").nullable();
table.string("linkDomain").nullable();
table.string("linkImage", 2000).nullable();
table.string("linkFavicon", 2000).nullable();
table.text("linkHtml").nullable();
table.boolean("linkIFrameAllowed").nullable().defaultTo(false);
2021-08-29 01:15:35 +03:00
table.jsonb("tags").nullable();
});
};
/* Migrate over to new columns (and denormalize tags) */
const defaultPhotos = [
"https://slate.host/static/a1.jpg",
`${Constants.gateways.ipfs}/bafkreigjofaa2wvmcoi5vbe3h32cbqh35d35wdhodfxwgmfky3gcur6s5i`,
`${Constants.gateways.ipfs}/bafkreiaycwt6m3jabhetfqnsb63z2lx65vzepp5ejk2b56vxypwxiet6c4`,
`${Constants.gateways.ipfs}/bafkreifvdvygknj66bfqdximxmxobqelwhd3xejiq3vfplhzkopcfdetrq`,
`${Constants.gateways.ipfs}/bafkreihm3srxvhfppm2dvldw224v4m3prcc3b6pwe3rtuxpsu6wwjpgzpa`,
`${Constants.gateways.ipfs}/bafkreiardkkfxj3ip373ee2tf6ffivjqclq7ionemt6pw55e6hv7ws5pvu`,
`${Constants.gateways.ipfs}/bafkreick3nscgixwfpq736forz7kzxvvhuej6kszevpsgmcubyhsx2pf7i`,
`${Constants.gateways.ipfs}/bafkreibf3hoiyuk2ywjyoy24ywaaclo4k5rz53flesvr5h4qjlyzxamozm`,
];
const defaultBody = ["A user of Slate.", "", "A slate."];
2021-08-29 01:15:35 +03:00
const migrateUserTable = async () => {
const users = await DB.select("id", "data").from("users");
let count = 0;
2021-08-29 01:15:35 +03:00
for (let user of users) {
if (count % 1000 === 0) {
console.log(count);
}
count += 1;
let { data } = user;
2021-08-29 01:15:35 +03:00
let newUser = {
name: data.name,
body: data.body.slice(0, 2000),
2021-08-29 01:15:35 +03:00
photo: data.photo,
textileKey: data.tokens?.api,
2021-08-29 01:15:35 +03:00
twitterUsername: data.twitter?.username,
twitterVerified: data.twitter?.verified,
};
if (defaultPhotos.includes(data.photo)) {
// console.log(`replaced default photo ${data.photo} with null`);
newUser.photo = null;
}
if (defaultBody.includes(data.body)) {
// console.log(`replaced default body with null`);
newUser.body = null;
}
// console.log({ data });
// console.log({ id: user.id, newUser });
const response = await DB.from("users").where("id", user.id).update(newUser).returning("*");
// console.log({ response });
2021-08-29 01:15:35 +03:00
}
};
const deleteEmptySlates = async () => {
let emptySlates = await DB("slates")
.select("id")
.whereNotExists(function () {
this.select("id").from("slate_files").whereRaw('"slate_files"."slateId" = "slates"."id"');
});
for (let slate of emptySlates) {
await Data.deleteSlateById(slate);
}
};
const migrateSlateTable = async () => {
const slateFiles = () =>
DB.raw("coalesce(json_agg(?? order by ?? asc) filter (where ?? is not null), '[]') as ??", [
"files",
"slate_files.createdAt",
"files.id",
"objects",
]);
const slates = await DB.select("slates.id", "slates.data", slateFiles())
.from("slates")
.leftJoin("slate_files", "slate_files.slateId", "=", "slates.id")
.leftJoin("files", "slate_files.fileId", "=", "files.id")
.groupBy("slates.id");
2021-08-29 01:15:35 +03:00
for (let slate of slates) {
let coverImage = Utilities.selectSlateCoverImage(slate.objects);
let { data } = slate;
2021-08-29 01:15:35 +03:00
let newSlate = {
name: data.name,
body: data.body ? data.body.slice(0, 2000) : null,
coverImage,
2021-08-29 01:15:35 +03:00
};
if (defaultBody.includes(data.body)) {
// console.log(`replaced default body with null`);
newSlate.body = null;
}
// console.log({ data });
// console.log({ newSlate });
2021-08-29 01:15:35 +03:00
const response = await DB.from("slates").where("id", slate.id).update(newSlate);
}
};
const migrateFileTable = async () => {
const files = await DB.select("id", "oldData").from("files").where({ size: 0 });
let count = 0;
2021-08-29 01:15:35 +03:00
for (let file of files) {
console.log(count);
count += 1;
2021-08-29 01:15:35 +03:00
let data = file.oldData;
2021-09-04 01:41:45 +03:00
if (!data) {
console.log(`no data for file with id ${file.id}`);
continue;
}
2021-08-29 01:15:35 +03:00
let newFile = {
name: data.name ? data.name.slice(0, 255) : null,
body: data.body ? data.body.slice(0, 2000) : null,
size: Math.min(data.size, 2147483647),
2021-08-29 01:15:35 +03:00
type: data.type,
blurhash: !data.blurhash || data.blurhash.length > 40 ? null : data.blurhash,
linkName: data.link?.name ? data.link?.name.slice(0, 255) : null,
linkBody: data.link?.body ? data.link?.body.slice(0, 2000) : null,
linkAuthor: data.link?.author ? data.link?.author.slice(0, 255) : null,
linkSource: data.link?.source ? data.link?.source.slice(0, 255) : null,
2021-08-29 01:15:35 +03:00
linkDomain: data.link?.domain,
linkImage: !data.link?.image || data.link?.image.length > 2000 ? null : data.link?.image,
linkFavicon: !data.link?.logo || data.link?.logo.length > 2000 ? null : data.link?.logo,
2021-08-29 01:15:35 +03:00
linkHtml: data.link?.html,
linkIFrameAllowed: data.link?.iFrameAllowed,
};
if (data.unity) {
newFile.data = { unity: data.unity };
}
// if (newFile.data) {
// console.log({ unity: newFile.data.unity });
// }
let { coverImage } = data;
2021-09-04 01:41:45 +03:00
if (coverImage) {
// console.log({ coverImage });
2021-09-04 01:41:45 +03:00
let newCoverImage = {
...coverImage.data,
2021-09-04 01:41:45 +03:00
...coverImage,
};
delete newCoverImage.data;
newFile.coverImage = newCoverImage;
// console.log({ newCoverImage });
2021-09-04 01:41:45 +03:00
}
2021-10-03 00:38:54 +03:00
// let tags = await DB.select("slates.id", "slates.slatename", "slates.name")
// .from("slates")
// .join("slate_files", "slate_files.slateId", "=", "slates.id")
// .where("slate_files.fileId", file.id);
let tags = await DB.select("id", "slatename", "name")
2021-08-29 01:15:35 +03:00
.from("slates")
2021-10-03 00:38:54 +03:00
.whereExists(function () {
this.select("id")
.from("slate_files")
.whereRaw('"slate_files"."slateId" = "slates"."id"')
.where({ "slate_files.fileId": file.id });
2021-10-03 00:38:54 +03:00
});
2021-08-29 01:15:35 +03:00
if (tags?.length) {
newFile.tags = JSON.stringify(tags);
2021-08-29 01:15:35 +03:00
}
// console.log({ newFile });
const response = await DB.from("files").where("id", file.id).update(newFile).returning("*");
console.log(response);
2021-08-29 01:15:35 +03:00
}
};
/* Delete the old data */
2021-08-29 01:15:35 +03:00
const deleteOldData = async () => {
2021-08-29 01:15:35 +03:00
await DB.schema.table("users", function (table) {
table.dropColumn("data");
});
await DB.schema.table("slates", function (table) {
table.dropColumn("data");
});
await DB.schema.table("files", function (table) {
table.dropColumn("oldData");
2021-08-29 01:15:35 +03:00
});
};
const runScript = async () => {
// await printUsersTable();
// await printSlatesTable();
// await printFilesTable();
// await createSurveysTable();
// await addUserTextileColumns();
// await addUserColumns();
// await addSlateColumns();
// await addFileColumns();
// await alterFileColumns();
await migrateUserTable();
// await deleteEmptySlates();
// await migrateSlateTable();
2021-09-27 01:38:54 +03:00
// await migrateFileTable();
// await deleteOldData();
console.log("SCRIPT FINISHED. HIT CTRL + C TO END");
2021-08-29 01:15:35 +03:00
};
runScript();
/*
Users
[
2021-08-29 04:36:48 +03:00
'data.name', -> 'name' MIGRATED
'data.body', -> 'body' MIGRATED
'data.photo', -> 'photo' MIGRATED
'data.status', -> 'onboarding.hidePrivacyAlert' MIGRATED
2021-09-25 04:20:28 +03:00
'data.tokens.api', -> 'textileKey' MIGRATED
'data.onboarding', -> 'onboarding' MIGRATED
'data.twitter.username', -> 'twitterUsername' MIGRATED
'data.twitter.verified', -> 'twitterVerified' MIGRATED
2021-08-29 01:15:35 +03:00
]
Slates
[
2021-08-29 04:36:48 +03:00
'data.name', -> 'name' MIGRATED
'data.body', -> 'body' MIGRATED
'data.preview', -> 'coverImage' MIGRATED
2021-08-29 01:15:35 +03:00
]
Files
[
2021-08-29 04:36:48 +03:00
'data.name', -> 'name' MIGRATED
2021-08-31 00:20:24 +03:00
'data.size', -> 'size' MIGRATED
'data.type', -> 'type' MIGRATED
'data.blurhash', -> 'blurhash' MIGRATED
'data.body', -> 'body' MIGRATED
'data.coverImage', -> 'coverImage' MIGRATED
'data.unity', -> 'data.unity' MIGRATED
'data.link.name', -> 'linkName' MIGRATED
'data.link.body', -> 'linkBody' MIGRATED
'data.link.author', -> 'linkAuthor' MIGRATED
'data.link.source', -> 'linkSource' MIGRATED
'data.link.domain', -> 'linkDomain' MIGRATED
'data.link.image', -> 'linkImage' MIGRATED
'data.link.logo', -> 'linkFavicon' MIGRATED
'data.link.html', -> 'linkHtml' MIGRATED
'data.link.iFrameAllowed', -> 'linkIFrameAllowed' MIGRATED
2021-08-29 01:15:35 +03:00
ADD 'tags'
]
*/