chore: simplify and remove some scripts (#4838)

This commit is contained in:
Dmitry Gozman 2020-12-28 16:19:28 -08:00 committed by GitHub
parent 068d8612a7
commit a1232b6980
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 70 additions and 165 deletions

View File

@ -18,16 +18,14 @@
"tsc": "tsc -p .",
"tsc-installer": "tsc -p ./src/install/tsconfig.json",
"doc": "node utils/doclint/cli.js",
"test-infra": "folio utils/doclint/check_public_api/test/testMissingDocs.js && folio utils/doclint/preprocessor/test.js",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && node utils/generate_types/ --check-clean && npm run test-types && npm run test-infra",
"clean": "rimraf lib && rimraf types",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && node utils/generate_types/ --check-clean && npm run test-types && folio utils/doclint/test/",
"clean": "rimraf lib",
"prepare": "node install-from-github.js",
"build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-api-json",
"watch": "node utils/watch.js",
"test-types": "node utils/generate_types/ && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && npm run typecheck-tests",
"test-types": "node utils/generate_types/ && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && tsc -p ./test/",
"generate-channels": "node utils/generate_channels.js",
"generate-api-json": "node utils/doclint/generateApiJson.js > docs/api.json",
"typecheck-tests": "tsc -p ./test/",
"roll-browser": "node utils/roll_browser.js",
"coverage": "node test/checkCoverage.js",
"check-deps": "node utils/check_deps.js",

View File

@ -1 +0,0 @@
output/

View File

@ -16,7 +16,7 @@
// @ts-check
const { parseArgument, renderMd, clone } = require('../../parse_md');
const { parseArgument, renderMd, clone } = require('../parse_md');
const Documentation = require('./Documentation');
/** @typedef {import('./Documentation').MarkdownNode} MarkdownNode */
@ -159,7 +159,7 @@ function patchSignatures(spec, signatures) {
}
/**
* @param {string} text
* @param {string} text
* @returns {string}
*/
function createLink(text) {

View File

@ -1,44 +0,0 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
class Message {
/**
* @param {string} type
* @param {string} text
*/
constructor(type, text) {
this.type = type;
this.text = text;
}
/**
* @param {string} text
* @return {!Message}
*/
static error(text) {
return new Message('error', text);
}
/**
* @param {string} text
* @return {!Message}
*/
static warning(text) {
return new Message('warning', text);
}
}
module.exports = Message;

View File

@ -1,30 +0,0 @@
# DocLint
**Doclint** is a small program that lints Playwright's documentation against
Playwright's source code.
Doclint works in a few steps:
1. Read sources in `lib/` folder, parse AST trees and extract public API
2. Read sources in `docs/` folder, render markdown to HTML, use playwright to traverse the HTML
and extract described API
3. Compare one API to another
Doclint is also responsible for general markdown checks, most notably for the table of contents
relevancy.
## Running
```bash
npm run doc
```
## Tests
Doclint has its own set of jasmine tests, located at `utils/doclint/test` folder.
To execute tests, run:
```bash
npm run test-doclint
```

View File

@ -21,14 +21,13 @@ const playwright = require('../../');
const fs = require('fs');
const path = require('path');
const Source = require('./Source');
const Message = require('./Message');
const { parseMd, renderMd, applyTemplates, clone } = require('./../parse_md');
const { spawnSync } = require('child_process');
const preprocessor = require('./preprocessor');
const mdBuilder = require('./check_public_api/MDBuilder');
const mdBuilder = require('./MDBuilder');
/** @typedef {import('./check_public_api/Documentation').MarkdownNode} MarkdownNode */
/** @typedef {import('./check_public_api/Documentation').Type} Type */
/** @typedef {import('./Documentation').MarkdownNode} MarkdownNode */
/** @typedef {import('./Documentation').Type} Type */
const PROJECT_DIR = path.join(__dirname, '..', '..');
const VERSION = require(path.join(PROJECT_DIR, 'package.json')).version;
@ -52,8 +51,8 @@ async function run() {
const docs = await Source.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
const mdSources = [readme, binReadme, api, contributing, ...docs];
/** @type {!Array<!Message>} */
const messages = [];
/** @type {!Array<string>} */
const errors = [];
let changedFiles = false;
const header = fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-header.md')).toString();
@ -128,20 +127,20 @@ async function run() {
// Documentation checks.
{
const browserVersions = await getBrowserVersions();
messages.push(...(await preprocessor.runCommands(mdSources, {
errors.push(...(await preprocessor.runCommands(mdSources, {
libversion: VERSION,
chromiumVersion: browserVersions.chromium,
firefoxVersion: browserVersions.firefox,
webkitVersion: browserVersions.webkit,
})));
messages.push(...preprocessor.autocorrectInvalidLinks(PROJECT_DIR, mdSources, getRepositoryFiles()));
errors.push(...preprocessor.autocorrectInvalidLinks(PROJECT_DIR, mdSources, getRepositoryFiles()));
for (const source of mdSources.filter(source => source.hasUpdatedText()))
messages.push(Message.warning(`WARN: updated ${source.projectPath()}`));
errors.push(`WARN: updated ${source.projectPath()}`);
const jsSources = await Source.readdir(path.join(PROJECT_DIR, 'src', 'client'), '', []);
const missingDocs = require('./check_public_api/missingDocs.js');
messages.push(...missingDocs(apiSpec, jsSources, path.join(PROJECT_DIR, 'src', 'client', 'api.ts')));
const missingDocs = require('./missingDocs.js');
errors.push(...missingDocs(apiSpec, jsSources, path.join(PROJECT_DIR, 'src', 'client', 'api.ts')));
for (const source of mdSources) {
if (!source.hasUpdatedText())
@ -152,31 +151,19 @@ async function run() {
}
// Report results.
const errors = messages.filter(message => message.type === 'error');
if (errors.length) {
console.log('DocLint Failures:');
for (let i = 0; i < errors.length; ++i) {
let error = errors[i].text;
error = error.split('\n').join('\n ');
const error = errors[i].split('\n').join('\n ');
console.log(` ${i + 1}) ${RED_COLOR}${error}${RESET_COLOR}`);
}
}
const warnings = messages.filter(message => message.type === 'warning');
if (warnings.length) {
console.log('DocLint Warnings:');
for (let i = 0; i < warnings.length; ++i) {
let warning = warnings[i].text;
warning = warning.split('\n').join('\n ');
console.log(` ${i + 1}) ${YELLOW_COLOR}${warning}${RESET_COLOR}`);
}
}
let clearExit = messages.length === 0;
let clearExit = errors.length === 0;
if (changedFiles) {
if (clearExit)
console.log(`${YELLOW_COLOR}Some files were updated.${RESET_COLOR}`);
clearExit = false;
}
console.log(`${errors.length} failures, ${warnings.length} warnings.`);
console.log(`${errors.length} failures.`);
const runningTime = Date.now() - startTime;
console.log(`DocLint Finished in ${runningTime / 1000} seconds`);
process.exit(clearExit ? 0 : 1);

View File

@ -17,7 +17,7 @@
const fs = require('fs');
const path = require('path');
const { parseMd, applyTemplates } = require('../parse_md');
const mdBuilder = require('./check_public_api/MDBuilder');
const mdBuilder = require('./MDBuilder');
const PROJECT_DIR = path.join(__dirname, '..', '..');
{

View File

@ -16,13 +16,12 @@
*/
const mdBuilder = require('./MDBuilder');
const Message = require('../Message');
const ts = require('typescript');
const EventEmitter = require('events');
const Documentation = require('./Documentation');
/**
* @return {!Array<!Message>}
* @return {!Array<string>}
*/
module.exports = function lint(api, jsSources, apiFileName) {
const documentation = mdBuilder(api, true).documentation;
@ -31,26 +30,26 @@ module.exports = function lint(api, jsSources, apiFileName) {
for (const [className, methods] of apiMethods) {
const docClass = documentation.classes.get(className);
if (!docClass) {
errors.push(Message.error(`Missing documentation for "${className}"`));
errors.push(`Missing documentation for "${className}"`);
continue;
}
for (const [methodName, params] of methods) {
const member = docClass.members.get(methodName);
if (!member) {
errors.push(Message.error(`Missing documentation for "${className}.${methodName}"`));
errors.push(`Missing documentation for "${className}.${methodName}"`);
continue;
}
const memberParams = paramsForMember(member);
for (const paramName of params) {
if (!memberParams.has(paramName))
errors.push(Message.error(`Missing documentation for "${className}.${methodName}.${paramName}"`));
errors.push(`Missing documentation for "${className}.${methodName}.${paramName}"`);
}
}
}
for (const cls of documentation.classesArray) {
const methods = apiMethods.get(cls.name);
if (!methods) {
errors.push(Message.error(`Documented "${cls.name}" not found in sources`));
errors.push(`Documented "${cls.name}" not found in sources`);
continue;
}
for (const member of cls.membersArray) {
@ -58,13 +57,13 @@ module.exports = function lint(api, jsSources, apiFileName) {
continue;
const params = methods.get(member.name);
if (!params) {
errors.push(Message.error(`Documented "${cls.name}.${member.name}" not found is sources`));
errors.push(`Documented "${cls.name}.${member.name}" not found is sources`);
continue;
}
const memberParams = paramsForMember(member);
for (const paramName of memberParams) {
if (!params.has(paramName))
errors.push(Message.error(`Documented "${cls.name}.${member.name}.${paramName}" not found is sources`));
errors.push(`Documented "${cls.name}.${member.name}.${paramName}" not found is sources`);
}
}
}

View File

@ -15,13 +15,12 @@
*/
const path = require('path');
const Message = require('../Message');
function runCommands(sources, {libversion, chromiumVersion, firefoxVersion, webkitVersion}) {
// Release version is everything that doesn't include "-".
const isReleaseVersion = !libversion.includes('-');
const messages = [];
const errors = [];
for (const source of sources) {
const text = source.text();
const commandStartRegex = /<!--\s*gen:([a-z-]+)\s*-->/ig;
@ -34,8 +33,8 @@ function runCommands(sources, {libversion, chromiumVersion, firefoxVersion, webk
commandEndRegex.lastIndex = commandStartRegex.lastIndex;
const end = commandEndRegex.exec(text);
if (!end) {
messages.push(Message.error(`Failed to find 'gen:stop' for command ${start[0]}`));
return messages;
errors.push(`Failed to find 'gen:stop' for command ${start[0]}`);
return errors;
}
const commandName = start[1];
const from = commandStartRegex.lastIndex;
@ -63,13 +62,13 @@ function runCommands(sources, {libversion, chromiumVersion, firefoxVersion, webk
newText = generateTableOfContentsForSuperclass(source.text(), 'class: ' + commandName.substring('toc-extends-'.length));
if (newText === null)
messages.push(Message.error(`Unknown command 'gen:${commandName}'`));
errors.push(`Unknown command 'gen:${commandName}'`);
else
sourceEdits.edit(from, to, newText);
}
sourceEdits.commit(messages);
sourceEdits.commit(errors);
}
return messages;
return errors;
};
function getTOCEntriesForText(text) {
@ -117,7 +116,7 @@ function autocorrectInvalidLinks(projectRoot, sources, allowedFilePaths) {
pathToHashLinks.set(source.filePath(), hashLinks);
}
const messages = [];
const errors = [];
for (const source of sources) {
const allRelativePaths = [];
for (const filepath of allowedFilePaths) {
@ -146,7 +145,7 @@ function autocorrectInvalidLinks(projectRoot, sources, allowedFilePaths) {
// Attempt to autocorrect
const newRelativePath = autocorrectText(relativePath, allRelativePaths);
if (!newRelativePath) {
messages.push(Message.error(`Bad link in ${source.projectPath()}:${lineNumber + 1}: file ${relativePath} does not exist`));
errors.push(`Bad link in ${source.projectPath()}:${lineNumber + 1}: file ${relativePath} does not exist`);
continue;
}
resolvedPath = resolveLinkPath(source, newRelativePath);
@ -161,15 +160,15 @@ function autocorrectInvalidLinks(projectRoot, sources, allowedFilePaths) {
if (newHashLink) {
sourceEdits.edit(hashOffset, hashOffset + hash.length, newHashLink);
} else {
messages.push(Message.error(`Bad link in ${source.projectPath()}:${lineNumber + 1}: hash "#${hash}" does not exist in "${path.relative(projectRoot, resolvedPath)}"`));
errors.push(`Bad link in ${source.projectPath()}:${lineNumber + 1}: hash "#${hash}" does not exist in "${path.relative(projectRoot, resolvedPath)}"`);
}
}
offset += line.length;
});
sourceEdits.commit(messages);
sourceEdits.commit(errors);
}
return messages;
return errors;
function resolveLinkPath(source, relativePath) {
if (!relativePath)
@ -190,19 +189,19 @@ class SourceEdits {
this._edits.push({from, to, newText});
}
commit(messages = []) {
commit(errors = []) {
if (!this._edits.length)
return;
this._edits.sort((a, b) => a.from - b.from);
for (const edit of this._edits) {
if (edit.from > edit.to) {
messages.push(Message.error('INTERNAL ERROR: incorrect edit!'));
errors.push('INTERNAL ERROR: incorrect edit!');
return;
}
}
for (let i = 0; i < this._edits.length - 1; ++i) {
if (this._edits[i].to > this._edits[i + 1].from) {
messages.push(Message.error('INTERNAL ERROR: edits are overlapping!'));
errors.push('INTERNAL ERROR: edits are overlapping!');
return;
}
}

View File

@ -18,9 +18,9 @@
const fs = require('fs');
const path = require('path');
const missingDocs = require('../missingDocs');
const Source = require('../../Source');
const Source = require('../Source');
const { folio } = require('folio');
const { parseMd } = require('../../../parse_md');
const { parseMd } = require('../../parse_md');
const { test, expect } = folio;
@ -30,8 +30,7 @@ test('missing docs', async ({}) => {
await Source.readFile(path.join(__dirname, 'test-api.ts')),
await Source.readFile(path.join(__dirname, 'test-api-class.ts')),
];
const messages = missingDocs(api, tsSources, path.join(__dirname, 'test-api.ts'));
const errors = messages.map(message => message.text);
const errors = missingDocs(api, tsSources, path.join(__dirname, 'test-api.ts'));
expect(errors).toEqual([
'Missing documentation for "Exists.exists2.extra"',
'Missing documentation for "Exists.exists2.options"',

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
const {runCommands} = require('.');
const {runCommands} = require('../preprocessor');
const Source = require('../Source');
const { folio } = require('folio');
const { describe, it, expect } = folio;
@ -34,19 +34,18 @@ describe('runCommands', function() {
const source = new Source('doc.md', `
<!-- gen:unknown-command -->something<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_REL);
const errors = runCommands([source], OPTIONS_REL);
expect(source.hasUpdatedText()).toBe(false);
expect(messages.length).toBe(1);
expect(messages[0].type).toBe('error');
expect(messages[0].text).toContain('Unknown command');
expect(errors.length).toBe(1);
expect(errors[0]).toContain('Unknown command');
});
describe('gen:version', function() {
it('should work', function() {
const source = new Source('doc.md', `
Playwright <!-- gen:version -->XXX<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
Playwright <!-- gen:version -->v1.3.0<!-- gen:stop -->
@ -56,8 +55,8 @@ describe('runCommands', function() {
const source = new Source('doc.md', `
Playwright <!-- gen:version -->XXX<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_DEV);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_DEV);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
Playwright <!-- gen:version -->Tip-Of-Tree<!-- gen:stop -->
@ -71,11 +70,10 @@ describe('runCommands', function() {
});
it('should not tolerate missing gen:stop', function() {
const source = new Source('doc.md', `<!--GEN:version-->`);
const messages = runCommands([source], OPTIONS_REL);
const errors = runCommands([source], OPTIONS_REL);
expect(source.hasUpdatedText()).toBe(false);
expect(messages.length).toBe(1);
expect(messages[0].type).toBe('error');
expect(messages[0].text).toContain(`Failed to find 'gen:stop'`);
expect(errors.length).toBe(1);
expect(errors[0]).toContain(`Failed to find 'gen:stop'`);
});
});
describe('gen:toc', function() {
@ -84,8 +82,8 @@ describe('runCommands', function() {
### class: page
#### page.$
#### page.$$`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`<!-- gen:toc -->
- [class: page](#class-page)
@ -104,8 +102,8 @@ describe('runCommands', function() {
# yo comment
\`\`\`
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`<!-- gen:toc -->
- [class: page](#class-page)
@ -121,8 +119,8 @@ describe('runCommands', function() {
const source = new Source('doc.md', `<!-- gen:toc -->XXX<!-- gen:stop -->
### some [link](#foobar) here
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`<!-- gen:toc -->
- [some link here](#some-link-here)
@ -139,8 +137,8 @@ describe('runCommands', function() {
#### first.2.1
## Second
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
## First
@ -161,8 +159,8 @@ describe('runCommands', function() {
<!-- gen:version -->xxx<!-- gen:stop -->
<!-- gen:version -->zzz<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
<!-- gen:version -->v1.3.0<!-- gen:stop -->
@ -174,8 +172,8 @@ describe('runCommands', function() {
const source = new Source('doc.md', `
Playwright <!-- gen:chromium-version -->XXX<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
Playwright <!-- gen:chromium-version -->80.0.4004.0<!-- gen:stop -->
@ -187,8 +185,8 @@ describe('runCommands', function() {
const source = new Source('doc.md', `
Playwright <!-- gen:firefox-version -->XXX<!-- gen:stop -->
`);
const messages = runCommands([source], OPTIONS_REL);
expect(messages.length).toBe(0);
const errors = runCommands([source], OPTIONS_REL);
expect(errors.length).toBe(0);
expect(source.hasUpdatedText()).toBe(true);
expect(source.text()).toBe(`
Playwright <!-- gen:firefox-version -->73.0b3<!-- gen:stop -->

View File

@ -17,7 +17,7 @@
//@ts-check
const path = require('path');
const {devices} = require('../..');
const Documentation = require('../doclint/check_public_api/Documentation');
const Documentation = require('../doclint/Documentation');
const PROJECT_DIR = path.join(__dirname, '..', '..');
const fs = require('fs');
const {parseOverrides} = require('./parseOverrides');
@ -39,7 +39,7 @@ let hadChanges = false;
const apiBody = parseMd(fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-body.md')).toString());
const apiParams = parseMd(fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-params.md')).toString());
const api = applyTemplates(apiBody, apiParams);
const mdResult = require('../doclint/check_public_api/MDBuilder')(api, true);
const mdResult = require('../doclint/MDBuilder')(api, true);
documentation = mdResult.documentation;
// Root module types are overridden.