upload: refactor and improvement and shared code paths between API and upload in app

components/sidebars/SidebarAddFileToBucket.js

@@ -55,7 +55,10 @@ export default class SidebarAddFileToBucket extends React.Component {
 
     await this.props.onSetFile({
       file,
-      slate: { id: this.props.data.slateId },
+      slate:
+        this.props.data && this.props.data.slateId
+          ? { id: this.props.data.slateId }
+          : null,
     });
   };
 

node_common/upload.js

@@ -0,0 +1,60 @@
+import * as Constants from "~/node_common/constants";
+import * as LibraryManager from "~/node_common/managers/library";
+import * as Utilities from "~/node_common/utilities";
+
+import FS from "fs-extra";
+import FORM from "formidable";
+
+export const formMultipart = (req, res, { user }) =>
+  new Promise((resolve, reject) => {
+    const f = new FORM.IncomingForm();
+    f.uploadDir = Constants.FILE_STORAGE_URL;
+    f.keepExtensions = true;
+    f.parse(req, async (e, fields, files) => {
+      if (e) {
+        return reject({
+          decorator: "SERVER_UPLOAD_PARSE_FAILURE",
+          error: true,
+          message: e,
+        });
+      }
+
+      if (!files.image) {
+        return reject({
+          decorator: "SERVER_UPLOAD_NOT_IMAGE_TYPE",
+          error: true,
+          message: files,
+        });
+      }
+
+      const path = files.image._writeStream.path;
+      const localPath = `./${path}`;
+      const data = LibraryManager.createLocalDataIncomplete(files.image);
+
+      const {
+        buckets,
+        bucketKey,
+        bucketName,
+      } = await Utilities.getBucketAPIFromUserToken(user.data.tokens.api);
+
+      let readFile;
+      let push;
+      // TODO(jim): Send this file to buckets.
+      try {
+        // NOTE(jim): Push pathPath to your bucket.
+        readFile = await FS.readFileSync(path).buffer;
+        push = await buckets.pushPath(bucketKey, data.name, readFile);
+      } catch (e) {
+        return reject({
+          decorator: "SERVER_BUCKETS_PUSH_ISSUE",
+          error: true,
+          message: e,
+        });
+      }
+
+      // NOTE(jim): Remove the file when you're done with it.
+      await FS.unlinkSync(localPath);
+
+      return resolve({ data, ipfs: push.path.path });
+    });
+  });

node_common/utilities.js

@@ -1,6 +1,7 @@
 import * as Environment from "~/node_common/environment";
 import * as Strings from "~/common/strings";
 import * as Powergate from "~/node_common/powergate";
+import * as Constants from "~/node_common/constants";
 
 import JWT from "jsonwebtoken";
 

pages/api/data/[upload].js

@@ -1,15 +1,13 @@
 import * as MW from "~/node_common/middleware";
-import * as Constants from "~/node_common/constants";
-import * as Data from "~/node_common/data";
+import * as Upload from "~/node_common/upload";
 import * as Utilities from "~/node_common/utilities";
+import * as Data from "~/node_common/data";
 import * as LibraryManager from "~/node_common/managers/library";
 
-import FORM from "formidable";
-import FS from "fs-extra";
-
 const initCORS = MW.init(MW.CORS);
 const initAuth = MW.init(MW.RequireCookieAuthentication);
 
+// NOTE(jim): To support multipart request.
 export const config = {
   api: {
     bodyParser: false,
@@ -20,70 +18,47 @@ export default async (req, res) => {
   initCORS(req, res);
   initAuth(req, res);
 
-  const f = new FORM.IncomingForm();
-  f.uploadDir = Constants.FILE_STORAGE_URL;
-  f.keepExtensions = true;
-  f.parse(req, async (e, fields, files) => {
-    if (e) {
-      return res
-        .status(500)
-        .send({ decorator: "SERVER_UPLOAD_PARSE_FAILURE", error: true });
-    }
+  const id = Utilities.getIdFromCookie(req);
+  const user = await Data.getUserById({
+    id,
+  });
 
-    if (!files.image) {
-      return res
-        .status(500)
-        .send({ decorator: "SERVER_UPLOAD_NOT_IMAGE_TYPE", error: true });
-    }
+  const response = await Upload.formMultipart(req, res, {
+    user,
+  });
 
-    const path = files.image._writeStream.path;
-    const data = LibraryManager.createLocalDataIncomplete(files.image);
+  if (!response) {
+    return res
+      .status(404)
+      .send({ decorator: "SERVER_UPLOAD_ERROR", error: true });
+  }
 
-    // TODO(jim): Send this file to buckets.
-    const id = Utilities.getIdFromCookie(req);
-    const user = await Data.getUserById({
-      id,
-    });
+  if (response.error) {
+    // NOTE(jim): To debug potential textile issues with matching CIDs.
+    console.log({ message: response.message });
+    return res
+      .status(500)
+      .send({ decorator: response.decorator, error: response.error });
+  }
 
-    const {
-      buckets,
-      bucketKey,
-      bucketName,
-    } = await Utilities.getBucketAPIFromUserToken(user.data.tokens.api);
+  const { data, ipfs } = response;
 
-    let readFile;
-    let push;
-    try {
-      // NOTE(jim): Push pathPath to your bucket.
-      readFile = await FS.readFileSync(path).buffer;
-      push = await buckets.pushPath(bucketKey, data.name, readFile);
-    } catch (e) {
-      console.log(e);
-      return res
-        .status(500)
-        .send({ decorator: "SERVER_BUCKETS_PUSH_ISSUE", error: true });
-    }
+  const finalData = LibraryManager.updateDataIPFS(data, {
+    ipfs,
+  });
 
-    // NOTE(jim): Update your user flag.
-    const updated = LibraryManager.updateDataIPFS(data, {
-      ipfs: push.path.path,
-    });
+  const updatedUserDataFields = LibraryManager.addData({
+    user,
+    data: finalData,
+  });
 
-    // NOTE(jim): Update your library
-    const updatedUserData = LibraryManager.addData({ user, data: updated });
+  await Data.updateUserById({
+    id: user.id,
+    data: updatedUserDataFields,
+  });
 
-    // NOTE(jim): Update your user
-    const response = await Data.updateUserById({
-      id: user.id,
-      data: updatedUserData,
-    });
-
-    // NOTE(jim): Remove the file when you're done with it.
-    await FS.unlinkSync(`./${path}`);
-
-    return res.status(200).send({
-      decorator: "SERVER_UPLOAD",
-      data: updated,
-    });
+  return res.status(200).send({
+    decorator: "SERVER_UPLOAD",
+    data: finalData,
   });
 };

pages/api/v1/upload-data/[id].js

@@ -1,15 +1,12 @@
 import * as MW from "~/node_common/middleware";
-import * as Constants from "~/node_common/constants";
 import * as Data from "~/node_common/data";
-import * as Utilities from "~/node_common/utilities";
 import * as LibraryManager from "~/node_common/managers/library";
 import * as Strings from "~/common/strings";
-
-import FORM from "formidable";
-import FS from "fs-extra";
+import * as Upload from "~/node_common/upload";
 
 const initCORS = MW.init(MW.CORS);
 
+// NOTE(jim): To support multipart request.
 export const config = {
   api: {
     bodyParser: false,
@@ -26,7 +23,6 @@ export default async (req, res) => {
     });
   }
 
-  // NOTE(jim): Get Slate to make sure we can add to it.
   let slate = await Data.getSlateById({ id: req.query.id });
 
   if (!slate) {
@@ -62,121 +58,96 @@ export default async (req, res) => {
     });
   }
 
-  const f = new FORM.IncomingForm();
-  f.uploadDir = Constants.FILE_STORAGE_URL;
-  f.keepExtensions = true;
-  f.parse(req, async (e, fields, files) => {
-    if (e) {
-      return res
-        .status(500)
-        .send({ decorator: "V1_SERVER_UPLOAD_PARSE_FAILURE", error: true });
-    }
+  const user = await Data.getUserById({
+    id: key.owner_id,
+  });
 
-    if (!files.image) {
-      return res
-        .status(500)
-        .send({ decorator: "V1_SERVER_UPLOAD_NOT_IMAGE_TYPE", error: true });
-    }
+  const uploadResponse = await Upload.formMultipart(req, res, {
+    user,
+  });
 
-    const path = files.image._writeStream.path;
-    const data = LibraryManager.createLocalDataIncomplete(files.image);
+  if (!uploadResponse) {
+    return res
+      .status(404)
+      .send({ decorator: "V1_SERVER_API_UPLOAD_ERROR", error: true });
+  }
 
-    // TODO(jim): Send this file to buckets.
-    const id = Utilities.getIdFromCookie(req);
-    const user = await Data.getUserById({
-      id: key.owner_id,
+  if (uploadResponse.error) {
+    // NOTE(jim): Just to debug potential textile issues with matching CIDs.
+    console.log({ message: uploadResponse.message });
+    return res.status(500).send({
+      decorator: uploadResponse.decorator,
+      error: uploadResponse.error,
     });
+  }
 
-    const {
-      buckets,
-      bucketKey,
-      bucketName,
-    } = await Utilities.getBucketAPIFromUserToken(user.data.tokens.api);
+  const { data, ipfs } = uploadResponse;
 
-    let readFile;
-    let push;
-    try {
-      // NOTE(jim): Push pathPath to your bucket.
-      readFile = await FS.readFileSync(path).buffer;
-      push = await buckets.pushPath(bucketKey, data.name, readFile);
-    } catch (e) {
-      console.log(e);
-      return res
-        .status(500)
-        .send({ decorator: "V1_SERVER_BUCKETS_PUSH_ISSUE", error: true });
-    }
+  const updatedData = LibraryManager.updateDataIPFS(data, {
+    ipfs,
+  });
 
-    // NOTE(jim): Update your user flag.
-    const updated = LibraryManager.updateDataIPFS(data, {
-      ipfs: push.path.path,
+  const updatedUserDataFields = LibraryManager.addData({
+    user,
+    data: updatedData,
+  });
+
+  await Data.updateUserById({
+    id: user.id,
+    data: updatedUserDataFields,
+  });
+
+  slate = await Data.getSlateById({ id: req.query.id });
+
+  if (!slate) {
+    return res.status(404).json({
+      decorator: "V1_SERVER_UPLOAD_SLATE_NOT_FOUND",
+      error: true,
     });
+  }
 
-    // NOTE(jim): Update your library
-    const updatedUserData = LibraryManager.addData({ user, data: updated });
-
-    // NOTE(jim): Update your user
-    const response = await Data.updateUserById({
-      id: user.id,
-      data: updatedUserData,
+  if (slate.error) {
+    return res.status(500).json({
+      decorator: "V1_SERVER_UPLOAD_SLATE_NOT_FOUND",
+      error: true,
     });
+  }
 
-    // NOTE(jim): Remove the file when you're done with it.
-    await FS.unlinkSync(`./${path}`);
+  const url = `https://hub.textile.io${updatedData.ipfs}`;
+  const newSlateObjectEntity = {
+    id: updatedData.id,
+    ownerId: user.id,
+    name: updatedData.name,
+    url,
+  };
 
-    // NOTE(jim): Make the call again to handle any time difference.
-    slate = await Data.getSlateById({ id: req.query.id });
+  const updatedSlate = await Data.updateSlateById({
+    id: slate.id,
+    updated_at: new Date(),
+    data: {
+      ...slate.data,
+      objects: [newSlateObjectEntity, ...slate.data.objects],
+    },
+  });
 
-    if (!slate) {
-      return res.status(404).json({
-        decorator: "V1_SERVER_UPLOAD_SLATE_NOT_FOUND",
-        error: true,
-      });
-    }
-
-    if (slate.error) {
-      return res.status(500).json({
-        decorator: "V1_SERVER_UPLOAD_SLATE_NOT_FOUND",
-        error: true,
-      });
-    }
-
-    // NOTE(jim): We have a key, we know the slate is real, so add to it.
-    const update = await Data.updateSlateById({
-      id: slate.id,
-      updated_at: new Date(),
-      data: {
-        ...slate.data,
-        objects: [
-          {
-            id: updated.id,
-            ownerId: user.id,
-            name: updated.name,
-            url: `https://hub.textile.io${updated.ipfs}`,
-          },
-          ...slate.data.objects,
-        ],
-      },
+  if (!updatedSlate) {
+    return res.status(500).json({
+      decorator: "V1_SERVER_UPLOAD_TO_SLATE_ERROR",
+      error: true,
     });
+  }
 
-    if (!update) {
-      return res.status(500).json({
-        decorator: "V1_SERVER_UPLOAD_TO_SLATE_ERROR",
-        error: true,
-      });
-    }
-
-    if (update.error) {
-      return res.status(500).json({
-        decorator: "V1_SERVER_UPLOAD_TO_SLATE_ERROR",
-        error: true,
-      });
-    }
-
-    return res.status(200).send({
-      decorator: "V1_UPLOAD_DATA_TO_SLATE",
-      data: updated,
-      slate: update,
-      url: `https://hub.textile.io${updated.ipfs}`,
+  if (updatedSlate.error) {
+    return res.status(500).json({
+      decorator: "V1_SERVER_UPLOAD_TO_SLATE_ERROR",
+      error: true,
     });
+  }
+
+  return res.status(200).send({
+    decorator: "V1_UPLOAD_DATA_TO_SLATE",
+    data: updatedData,
+    slate: updatedSlate,
+    url,
   });
 };