From 806a71a4f09c7e737adfec74ab65bf3d73fefdcd Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 16 Sep 2021 17:48:43 -0700 Subject: [PATCH] feat(fetch): support form data and json encodings (#8975) --- docs/src/api/class-fetchrequest.md | 14 ++- src/client/fetch.ts | 75 ++++++++++-- src/dispatchers/networkDispatchers.ts | 2 +- src/protocol/channels.ts | 2 + src/protocol/protocol.yml | 1 + src/protocol/validator.ts | 1 + src/server/fetch.ts | 39 ++++++- src/server/formData.ts | 90 +++++++++++++++ src/server/types.ts | 13 +++ src/utils/utils.ts | 4 + tests/browsercontext-fetch.spec.ts | 157 +++++++++++++++++++++++++- types/types.d.ts | 18 ++- 12 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 src/server/formData.ts diff --git a/docs/src/api/class-fetchrequest.md b/docs/src/api/class-fetchrequest.md index 82d4f2affd..3bef765ffd 100644 --- a/docs/src/api/class-fetchrequest.md +++ b/docs/src/api/class-fetchrequest.md @@ -39,9 +39,12 @@ If set changes the fetch method (e.g. PUT or POST). If not specified, GET method Allows to set HTTP headers. ### option: FetchRequest.fetch.data -- `data` <[string]|[Buffer]> +- `data` <[string]|[Buffer]|[Serializable]> -Allows to set post data of the fetch. +Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way: +* If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form using `application/x-www-form-urlencoded` encoding. +* If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using `multipart/form-data` encoding. +* Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`. ### option: FetchRequest.fetch.timeout - `timeout` <[float]> @@ -108,9 +111,12 @@ Query parameters to be send with the URL. Allows to set HTTP headers. ### option: FetchRequest.post.data -- `data` <[string]|[Buffer]> +- `data` <[string]|[Buffer]|[Serializable]> -Allows to set post data of the fetch. +Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way: +* If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form using `application/x-www-form-urlencoded` encoding. +* If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using `multipart/form-data` encoding. +* Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`. ### option: FetchRequest.post.timeout - `timeout` <[float]> diff --git a/src/client/fetch.ts b/src/client/fetch.ts index 1e5a735701..09da722c10 100644 --- a/src/client/fetch.ts +++ b/src/client/fetch.ts @@ -14,21 +14,25 @@ * limitations under the License. */ +import { ReadStream } from 'fs'; +import path from 'path'; +import * as mime from 'mime'; +import { Serializable } from '../../types/structs'; import * as api from '../../types/types'; import { HeadersArray } from '../common/types'; import * as channels from '../protocol/channels'; import { kBrowserOrContextClosedError } from '../utils/errors'; -import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils'; +import { assert, headersObjectToArray, isFilePayload, isString, objectToArray } from '../utils/utils'; import { ChannelOwner } from './channelOwner'; import * as network from './network'; import { RawHeaders } from './network'; -import { Headers } from './types'; +import { FilePayload, Headers } from './types'; export type FetchOptions = { params?: { [key: string]: string; }, method?: string, headers?: Headers, - data?: string | Buffer, + data?: string | Buffer | Serializable, timeout?: number, failOnStatusCode?: boolean, }; @@ -67,7 +71,7 @@ export class FetchRequest extends ChannelOwner { @@ -87,9 +91,34 @@ export class FetchRequest extends ChannelOwner { + const buffer = await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', chunk => chunks.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + stream.on('error', err => reject(err)); + }); + const streamPath: string = Buffer.isBuffer(stream.path) ? stream.path.toString('utf8') : stream.path; + return { + name: path.basename(streamPath), + mimeType: mime.getType(streamPath) || 'application/octet-stream', + buffer: buffer.toString('base64'), + }; +} \ No newline at end of file diff --git a/src/dispatchers/networkDispatchers.ts b/src/dispatchers/networkDispatchers.ts index 057b5b8bc6..ff2e6a0825 100644 --- a/src/dispatchers/networkDispatchers.ts +++ b/src/dispatchers/networkDispatchers.ts @@ -188,6 +188,7 @@ export class FetchRequestDispatcher extends Dispatcher Validator): Scheme { method: tOptional(tString), headers: tOptional(tArray(tType('NameValue'))), postData: tOptional(tBinary), + formData: tOptional(tAny), timeout: tOptional(tNumber), failOnStatusCode: tOptional(tBoolean), }); diff --git a/src/server/fetch.ts b/src/server/fetch.ts index afadba3389..1ba6d6fe46 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -22,12 +22,13 @@ import * as https from 'https'; import { BrowserContext } from './browserContext'; import * as types from './types'; import { pipeline, Readable, Transform } from 'stream'; -import { createGuid, monotonicTime } from '../utils/utils'; +import { createGuid, isFilePayload, monotonicTime } from '../utils/utils'; import { SdkObject } from './instrumentation'; import { Playwright } from './playwright'; import { HeadersArray, ProxySettings } from './types'; import { HTTPCredentials } from '../../types/types'; import { TimeoutSettings } from '../utils/timeoutSettings'; +import { MultipartFormData } from './formData'; type FetchRequestOptions = { @@ -130,7 +131,13 @@ export abstract class FetchRequest extends SdkObject { requestUrl.searchParams.set(name, value); } - const fetchResponse = await this._sendRequest(requestUrl, options, params.postData); + let postData; + if (['POST', 'PUSH', 'PATCH'].includes(method)) + postData = params.formData ? serilizeFormData(params.formData, headers) : params.postData; + else if (params.postData || params.formData) + throw new Error(`Method ${method} does not accept post data`); + + const fetchResponse = await this._sendRequest(requestUrl, options, postData); const fetchUid = this._storeResponseBody(fetchResponse.body); if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400)) return { error: `${fetchResponse.status} ${fetchResponse.statusText}` }; @@ -410,3 +417,31 @@ function parseCookie(header: string) { } return cookie; } + +function serilizeFormData(data: any, headers: { [name: string]: string }): Buffer { + const contentType = headers['content-type'] || 'application/json'; + if (contentType === 'application/json') { + const json = JSON.stringify(data); + headers['content-type'] ??= contentType; + return Buffer.from(json, 'utf8'); + } else if (contentType === 'application/x-www-form-urlencoded') { + const searchParams = new URLSearchParams(); + for (const [name, value] of Object.entries(data)) + searchParams.append(name, String(value)); + return Buffer.from(searchParams.toString(), 'utf8'); + } else if (contentType === 'multipart/form-data') { + const formData = new MultipartFormData(); + for (const [name, value] of Object.entries(data)) { + if (isFilePayload(value)) { + const payload = value as types.FilePayload; + formData.addFileField(name, payload); + } else if (value !== undefined) { + formData.addField(name, String(value)); + } + } + headers['content-type'] = formData.contentTypeHeader(); + return formData.finish(); + } else { + throw new Error(`Cannot serialize data using content type: ${contentType}`); + } +} diff --git a/src/server/formData.ts b/src/server/formData.ts new file mode 100644 index 0000000000..b58d31a860 --- /dev/null +++ b/src/server/formData.ts @@ -0,0 +1,90 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as types from './types'; + +export class MultipartFormData { + private readonly _boundary: string; + private readonly _chunks: Buffer[] = []; + + constructor() { + this._boundary = generateUniqueBoundaryString(); + } + + contentTypeHeader() { + return `multipart/form-data; boundary=${this._boundary}`; + } + + addField(name: string, value: string) { + this._beginMultiPartHeader(name); + this._finishMultiPartHeader(); + this._chunks.push(Buffer.from(value)); + this._finishMultiPartField(); + } + + addFileField(name: string, value: types.FilePayload) { + this._beginMultiPartHeader(name); + this._chunks.push(Buffer.from(`; filename="${value.name}"`)); + this._chunks.push(Buffer.from(`\r\ncontent-type: ${value.mimeType || 'application/octet-stream'}`)); + this._finishMultiPartHeader(); + this._chunks.push(Buffer.from(value.buffer, 'base64')); + this._finishMultiPartField(); + } + + finish(): Buffer { + this._addBoundary(true); + return Buffer.concat(this._chunks); + } + + private _beginMultiPartHeader(name: string) { + this._addBoundary(); + this._chunks.push(Buffer.from(`content-disposition: form-data; name="${name}"`)); + } + + private _finishMultiPartHeader() { + this._chunks.push(Buffer.from(`\r\n\r\n`)); + } + + private _finishMultiPartField() { + this._chunks.push(Buffer.from(`\r\n`)); + } + + private _addBoundary(isLastBoundary?: boolean) { + this._chunks.push(Buffer.from('--' + this._boundary)); + if (isLastBoundary) + this._chunks.push(Buffer.from('--')); + this._chunks.push(Buffer.from('\r\n')); + } +} + +const alphaNumericEncodingMap = [ + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, + 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, + 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, + 0x77, 0x78, 0x79, 0x7A, 0x30, 0x31, 0x32, 0x33, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42 +]; + +// See generateUniqueBoundaryString() in WebKit +function generateUniqueBoundaryString(): string { + const charCodes = []; + for (let i = 0; i < 16; i++) + charCodes.push(alphaNumericEncodingMap[Math.floor(Math.random() * alphaNumericEncodingMap.length)]); + return '----WebKitFormBoundary' + String.fromCharCode(...charCodes); +} diff --git a/src/server/types.ts b/src/server/types.ts index a34104a0f8..8789898b5b 100644 --- a/src/server/types.ts +++ b/src/server/types.ts @@ -372,12 +372,25 @@ export type SetStorageState = { origins?: OriginStorage[] }; +export type FileInfo = { + name: string, + mimeType?: string, + buffer: Buffer, +}; + +export type FormField = { + name: string, + value?: string, + file?: FileInfo, +}; + export type FetchOptions = { url: string, params?: { [name: string]: string }, method?: string, headers?: { [name: string]: string }, postData?: Buffer, + formData?: FormField[], timeout?: number, failOnStatusCode?: boolean, }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index f6c369f01b..4ba1abb6ea 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -411,3 +411,7 @@ export function wrapInASCIIBox(text: string, padding = 0): string { '╚' + '═'.repeat(maxLength + padding * 2) + '╝', ].join('\n'); } + +export function isFilePayload(value: any): boolean { + return typeof value === 'object' && value['name'] && value['mimeType'] && value['buffer']; +} diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts index 7eb35de564..a1b0b73a40 100644 --- a/tests/browsercontext-fetch.spec.ts +++ b/tests/browsercontext-fetch.spec.ts @@ -14,8 +14,10 @@ * limitations under the License. */ +import formidable from 'formidable'; import http from 'http'; import zlib from 'zlib'; +import fs from 'fs'; import { pipeline } from 'stream'; import { contextTest as it, expect } from './config/browserTest'; import { suppressCertificateWarning } from './config/utils'; @@ -166,7 +168,7 @@ for (const method of ['get', 'post', 'fetch']) { const error = await context._request[method](server.PREFIX + '/does-not-exist.html', { failOnStatusCode: true }).catch(e => e); - expect(error.message).toContain('Request failed: 404 Not Found'); + expect(error.message).toContain('404 Not Found'); }); } @@ -705,3 +707,156 @@ it('should override request parameters', async function({context, page, server}) expect(req.headers.foo).toBe('bar'); expect((await req.postBody).toString('utf8')).toBe('data'); }); + +it('should support application/x-www-form-urlencoded', async function({context, page, server}) { + const [req] = await Promise.all([ + server.waitForRequest('/empty.html'), + context._request.post(server.EMPTY_PAGE, { + headers: { + 'content-type': 'application/x-www-form-urlencoded' + }, + data: { + firstName: 'John', + lastName: 'Doe', + file: 'f.js', + } + }) + ]); + expect(req.method).toBe('POST'); + expect(req.headers['content-type']).toBe('application/x-www-form-urlencoded'); + const body = (await req.postBody).toString('utf8'); + const params = new URLSearchParams(body); + expect(params.get('firstName')).toBe('John'); + expect(params.get('lastName')).toBe('Doe'); + expect(params.get('file')).toBe('f.js'); +}); + +it('should encode to application/json by default', async function({context, page, server}) { + const data = { + firstName: 'John', + lastName: 'Doe', + file: { + name: 'f.js' + }, + }; + const [req] = await Promise.all([ + server.waitForRequest('/empty.html'), + context._request.post(server.EMPTY_PAGE, { data }) + ]); + expect(req.method).toBe('POST'); + expect(req.headers['content-type']).toBe('application/json'); + const body = (await req.postBody).toString('utf8'); + const json = JSON.parse(body); + expect(json).toEqual(data); +}); + +it('should support multipart/form-data', async function({context, page, server}) { + const formReceived = new Promise(resolve => { + server.setRoute('/empty.html', async (serverRequest, res) => { + const form = new formidable.IncomingForm(); + form.parse(serverRequest, (error, fields, files) => { + server.serveFile(serverRequest, res); + resolve({error, fields, files, serverRequest }); + }); + }); + }); + + const file = { + name: 'f.js', + mimeType: 'text/javascript', + buffer: Buffer.from('var x = 10;\r\n;console.log(x);') + }; + const [{error, fields, files, serverRequest}, response] = await Promise.all([ + formReceived, + context._request.post(server.EMPTY_PAGE, { + headers: { + 'content-type': 'multipart/form-data' + }, + data: { + firstName: 'John', + lastName: 'Doe', + file + } + }) + ]); + expect(error).toBeFalsy(); + expect(serverRequest.method).toBe('POST'); + expect(serverRequest.headers['content-type']).toContain('multipart/form-data'); + expect(fields['firstName']).toBe('John'); + expect(fields['lastName']).toBe('Doe'); + expect(files['file'].name).toBe(file.name); + expect(files['file'].type).toBe(file.mimeType); + expect(fs.readFileSync(files['file'].path).toString()).toBe(file.buffer.toString('utf8')); + expect(response.status()).toBe(200); +}); + +it('should support multipart/form-data with ReadSream values', async function({context, page, asset, server}) { + const formReceived = new Promise(resolve => { + server.setRoute('/empty.html', async (serverRequest, res) => { + const form = new formidable.IncomingForm(); + form.parse(serverRequest, (error, fields, files) => { + server.serveFile(serverRequest, res); + resolve({error, fields, files, serverRequest }); + }); + }); + }); + const readStream = fs.createReadStream(asset('simplezip.json')); + const [{error, fields, files, serverRequest}, response] = await Promise.all([ + formReceived, + context._request.post(server.EMPTY_PAGE, { + headers: { + 'content-type': 'multipart/form-data' + }, + data: { + firstName: 'John', + lastName: 'Doe', + readStream + } + }) + ]); + expect(error).toBeFalsy(); + expect(serverRequest.method).toBe('POST'); + expect(serverRequest.headers['content-type']).toContain('multipart/form-data'); + expect(fields['firstName']).toBe('John'); + expect(fields['lastName']).toBe('Doe'); + expect(files['readStream'].name).toBe('simplezip.json'); + expect(files['readStream'].type).toBe('application/json'); + expect(fs.readFileSync(files['readStream'].path).toString()).toBe(fs.readFileSync(asset('simplezip.json')).toString()); + expect(response.status()).toBe(200); +}); + +it('should throw nice error on unsupported encoding', async function({context, server}) { + const error = await context._request.post(server.EMPTY_PAGE, { + headers: { + 'content-type': 'unknown' + }, + data: { + firstName: 'John', + lastName: 'Doe', + } + }).catch(e => e); + expect(error.message).toContain('Cannot serialize data using content type: unknown'); +}); + +it('should throw nice error on unsupported data type', async function({context, server}) { + const error = await context._request.post(server.EMPTY_PAGE, { + headers: { + 'content-type': 'application/json' + }, + data: () => true + }).catch(e => e); + expect(error.message).toContain(`Unexpected 'data' type`); +}); + +it('should throw when data passed for unsupported request', async function({context, server}) { + const error = await context._request.fetch(server.EMPTY_PAGE, { + method: 'GET', + headers: { + 'content-type': 'application/json' + }, + data: { + foo: 'bar' + } + }).catch(e => e); + expect(error.message).toContain(`Method GET does not accept post data`); +}); diff --git a/types/types.d.ts b/types/types.d.ts index 7d824231b8..67a4ae6b60 100644 --- a/types/types.d.ts +++ b/types/types.d.ts @@ -12646,9 +12646,14 @@ export interface FetchRequest { */ fetch(urlOrRequest: string|Request, options?: { /** - * Allows to set post data of the fetch. + * Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way: + * - If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form + * using `application/x-www-form-urlencoded` encoding. + * - If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using + * `multipart/form-data` encoding. + * - Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`. */ - data?: string|Buffer; + data?: string|Buffer|Serializable; /** * Whether to throw on response codes other than 2xx and 3xx. By default response object is returned for all status codes. @@ -12712,9 +12717,14 @@ export interface FetchRequest { */ post(urlOrRequest: string|Request, options?: { /** - * Allows to set post data of the fetch. + * Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way: + * - If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form + * using `application/x-www-form-urlencoded` encoding. + * - If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using + * `multipart/form-data` encoding. + * - Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`. */ - data?: string|Buffer; + data?: string|Buffer|Serializable; /** * Whether to throw on response codes other than 2xx and 3xx. By default response object is returned for all status codes.