filecoin-project/slate
1
1
Fork 0
mirror of https://github.com/filecoin-project/slate.git synced 2024-12-24 17:44:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

uploads: removes readStreams and watermark, sends data straight to textile buckets and keeps nothing local.

Browse Source
This commit is contained in:
jimmylee 2020-08-15 16:42:26 -07:00
parent 9ad4f7bfce
commit 4007910627
1 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
node_common/upload.js
View File

@ -1,17 +1,39 @@
import * as Constants from "~/node_common/constants";
import * as LibraryManager from "~/node_common/managers/library"; import * as LibraryManager from "~/node_common/managers/library";
import * as Utilities from "~/node_common/utilities"; import * as Utilities from "~/node_common/utilities";
import FS from "fs-extra";
import FORM from "formidable"; import FORM from "formidable";
// TODO(jim): Ideally we never keep the file locally. import { PassThrough } from "stream";
export const formMultipart = (req, res, { user }) => export const formMultipart = (req, res, { user }) =>
new Promise((resolve, reject) => { new Promise((resolve, reject) => {
const f = new FORM.IncomingForm(); const f = new FORM.IncomingForm();
f.uploadDir = Constants.FILE_STORAGE_URL; const p = new PassThrough();
const file = {};
f.keepExtensions = true; f.keepExtensions = true;
f.parse(req, async (e, fields, files) => {
f.onPart = (part) => {
if (!part.filename) {
form.handlePart(part);
return;
}
file.name = part.filename;
file.type = part.mime;
part.on("data", function (buffer) {
p.write(buffer);
});
part.on("end", function (data) {
p.end();
});
};
f.parse(req, async (e) => {
const { buckets, bucketKey } = await Utilities.getBucketAPIFromUserToken(user.data.tokens.api);
if (e) { if (e) {
return reject({ return reject({
decorator: "SERVER_UPLOAD_PARSE_FAILURE", decorator: "SERVER_UPLOAD_PARSE_FAILURE",
@ -20,40 +42,39 @@ export const formMultipart = (req, res, { user }) =>
}); });
} }
if (!files.data) { if (!file && !file.name) {
return reject({ return reject({
decorator: "SERVER_UPLOAD_ERROR_CHECK_FORM_TYPE", decorator: "SERVER_UPLOAD_ERROR_CHECK_FORM_TYPE",
error: true, error: true,
message: files, message: file,
}); });
} }
const path = files.data._writeStream.path; const data = LibraryManager.createLocalDataIncomplete(file);
const localPath = `./${path}`;
const data = LibraryManager.createLocalDataIncomplete(files.data);
const { buckets, bucketKey, bucketName } = await Utilities.getBucketAPIFromUserToken(user.data.tokens.api);
let readFile;
let push; let push;
try { try {
readFile = FS.createReadStream(path, { push = await buckets.pushPath(bucketKey, data.name, p);
highWaterMark: 1024 * 1024 * 3,
});
push = await buckets.pushPath(bucketKey, data.name, readFile);
} catch (e) { } catch (e) {
await FS.unlinkSync(localPath);
return reject({ return reject({
decorator: "SERVER_BUCKETS_PUSH_ISSUE", decorator: "SERVER_BUCKETS_PUSH_ISSUE",
error: true, error: true,
message: e, message: e,
}); });
} }
// NOTE(jim): Remove the file when you're done with it.
await FS.unlinkSync(localPath);
return resolve({ data, ipfs: push.path.path }); let ipfs = push.path.path;
try {
const newUpload = await buckets.listIpfsPath(ipfs);
data.size = newUpload.size;
} catch (e) {
return reject({
decorator: "SERVER_BUCKETS_VERIFY_ISSUE",
error: true,
message: e,
});
}
return resolve({ data, ipfs });
}); });
}); });