cleanup: keeps timeouts and modifications tidy for file uploads

common/file-utilities.js

@@ -34,12 +34,14 @@ export const upload = async ({ file, context, bucketName }) => {
   const _privateUploadMethod = (path, file) =>
     new Promise((resolve, reject) => {
       const XHR = new XMLHttpRequest();
+
       window.addEventListener(
         `cancel-${file.lastModified}-${file.name}`,
         () => {
           XHR.abort();
         }
       );
+
       XHR.open("post", path, true);
       XHR.onerror = (event) => {
         console.log(event);

pages/api/data/[upload].js

@@ -2,10 +2,12 @@ import * as Upload from "~/node_common/upload";
 import * as Utilities from "~/node_common/utilities";
 import * as Data from "~/node_common/data";
 import * as LibraryManager from "~/node_common/managers/library";
+import * as Strings from "~/common/strings";
 
 export const config = {
   api: {
     bodyParser: false,
+    externalResolver: true,
   },
 };
 
@@ -22,30 +24,30 @@ export default async (req, res) => {
       .send({ decorator: "UPLOAD_NOT_ALLOWED", error: true });
   }
 
-  console.log(`[upload] upload for ${user.username} started`);
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
+
   const response = await Upload.formMultipart(req, res, {
     user,
   });
-  console.log(`[upload] upload for ${user.username} responded`);
+
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
 
   if (!response) {
-    console.log(`[upload] upload for ${user.username} unsuccessful`);
     return res
       .status(404)
       .send({ decorator: "SERVER_UPLOAD_ERROR", error: true });
   }
 
   if (response.error) {
-    // NOTE(jim): To debug potential textile issues with matching CIDs.
-    console.log(`[upload] upload for ${user.username} unsuccessful`);
-    console.log({ message: response.message });
     return res
       .status(500)
       .send({ decorator: response.decorator, error: response.error });
   }
 
-  console.log(`[upload] upload for ${user.username} successful`);
-
   const { data, ipfs } = response;
 
   const finalData = LibraryManager.updateDataIPFS(data, {

pages/api/data/deal/[upload].js

@@ -2,6 +2,7 @@ import * as Upload from "~/node_common/upload";
 import * as Utilities from "~/node_common/utilities";
 import * as Data from "~/node_common/data";
 import * as LibraryManager from "~/node_common/managers/library";
+import * as Strings from "~/common/strings";
 
 // NOTE(jim): To support multipart request.
 const STAGING_DEAL_BUCKET = "stage-deal";
@@ -9,6 +10,7 @@ const STAGING_DEAL_BUCKET = "stage-deal";
 export const config = {
   api: {
     bodyParser: false,
+    externalResolver: true,
   },
 };
 
@@ -25,11 +27,19 @@ export default async (req, res) => {
       .send({ decorator: "UPLOAD_NOT_ALLOWED", error: true });
   }
 
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
+
   const response = await Upload.formMultipart(req, res, {
     user,
     bucketName: STAGING_DEAL_BUCKET,
   });
 
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
+
   if (!response) {
     return res
       .status(404)

pages/api/v1/upload-data/[id].js

@@ -3,6 +3,7 @@ import * as Constants from "~/node_common/constants";
 import * as LibraryManager from "~/node_common/managers/library";
 import * as Strings from "~/common/strings";
 import * as Upload from "~/node_common/upload";
+import * as Strings from "~/common/strings";
 
 const generateLayout = (items) => {
   if (!items) {
@@ -32,6 +33,7 @@ const generateLayout = (items) => {
 export const config = {
   api: {
     bodyParser: false,
+    externalResolver: true,
   },
 };
 
@@ -82,10 +84,18 @@ export default async (req, res) => {
     id: key.owner_id,
   });
 
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
+
   const uploadResponse = await Upload.formMultipart(req, res, {
     user,
   });
 
+  console.log(
+    `[ memory usage ] ${Strings.bytesToSize(process.memoryUsage().heapUsed)}`
+  );
+
   if (!uploadResponse) {
     return res
       .status(404)

server.js

@@ -36,29 +36,37 @@ app.prepare().then(async () => {
   );
 
   server.use("/public", express.static("public"));
-
-  server.get("/please-dont-use-timeout", async (r, s) => {
-    console.log("[ forbidden ] someone is using your testing timeout");
-
-    await sleep(2 * 60 * 1000);
-
-    return s
-      .status(200)
-      .json({ decorator: "SERVER_TIMEOUT_TEST", timeout: 2 * 60 * 1000 });
-  });
-
   server.get("/system", async (r, s) => s.redirect("/_/system"));
   server.get("/experiences", async (r, s) => s.redirect("/_/system"));
   server.get("/_/experiences", async (r, s) => s.redirect("/_/system"));
   server.get("/system/:c", async (r, s) =>
     s.redirect(`/_/system/${r.params.c}`)
   );
+
   server.get("/experiences/:m", async (r, s) =>
     s.redirect(`/_/experiences/${r.params.m}`)
   );
+
   server.all("/api/:a", async (r, s, next) => {
     return handler(r, s, r.url);
   });
+
+  // NOTE(jim): Upload data
+  server.post("/api/data/:b", async (req, res) => {
+    // NOTE(jim): Let this request take as long as it needs to.
+    req.setTimeout(0);
+
+    return await handler(req, res, req.url);
+  });
+
+  // NOTE(jim): Upload data for deal
+  server.post("/api/data/deal/:b", async (req, res) => {
+    // NOTE(jim): Let this request take as long as it needs to.
+    req.setTimeout(0);
+
+    return await handler(req, res, req.url);
+  });
+
   server.all("/api/:a/:b", async (r, s, next) => {
     return handler(r, s, r.url);
   });
@@ -260,6 +268,4 @@ app.prepare().then(async () => {
 
     console.log(`[ slate ] client: http://localhost:${Environment.PORT}`);
   });
-
-  listenServer.setTimeout(15 * 60 * 1000);
 });