dance/meta.ts
2022-11-21 20:20:35 +01:00

748 lines
22 KiB
TypeScript

import * as assert from "assert";
import * as fs from "fs/promises";
import * as G from "glob";
import * as path from "path";
const verbose = process.argv.includes("--verbose");
const moduleCommentRe =
new RegExp(String.raw`\/\*\*\n` // start of doc comment
+ String.raw`((?: \*(?:\n| .+\n))+?)` // #1: doc comment
+ String.raw` \*\/\n` // end of doc comment
+ String.raw`declare module \"(.+?)\"`, // #2: module name
"m");
const docCommentRe =
new RegExp(String.raw`^( *)` // #1: indentation
+ String.raw`\/\*\*\n` // start of doc comment
+ String.raw`((?:\1 \*(?:\n| .+\n))+?)` // #2: doc comment
+ String.raw`\1 \*\/\n` // end of doc comment
+ String.raw`\1export (?:async )?function (\w+)` // #3: function name
+ String.raw`(?:<[^>)\n]+>)?` // generic arguments
+ String.raw`\((.*|\n[\s\S]+?^\1)\)` // #4: parameters
+ String.raw`(?:: )?(.+)[;{]$`, // #5: return type (optional)
"gm");
function countNewLines(text: string) {
let count = 0;
for (let i = 0; i < text.length; i++) {
if (text.charCodeAt(i) === 10 /* \n */) {
count++;
}
}
return count;
}
const keyMapping: Record<string, keyof Builder.AdditionalCommand> = {
Command: "commands",
Commands: "commands",
Identifier: "identifier",
Identifiers: "identifier",
Keys: "keys",
Keybinding: "keys",
Keybindings: "keys",
Title: "title",
};
const valueConverter: Record<keyof Builder.AdditionalCommand, (x: string) => string> = {
commands(commands) {
return commands
.replace(/^`+|`+$/g, "")
.replace(/ +/g, " ")
.replace(/\.{3}(?= })/g, () =>
"-" + [...commands.matchAll(/(?<=\+)([a-zA-Z,]+)/g)].map((x) => x[0]).join(","))
.replace(/-([a-zA-Z,]*)(?= })/g, (_, exclude) =>
`$exclude: ${exclude === "" ? "[]" : JSON.stringify(exclude.split(","))}`)
.replace(/\+([a-zA-Z,]+)(?= })/g, (_, include) =>
`$include: ${JSON.stringify(include.split(","))}`)
.replace(/MAX_INT/g, `${2 ** 31 - 1}`); // Max integer supported in JSON.
},
identifier(identifier) {
return identifier.replace(/^`+|`+$/g, "");
},
keys(keys) {
return keys;
},
title(title) {
return title;
},
qualifiedIdentifier(qualifiedIdentifier) {
return qualifiedIdentifier;
},
line() {
throw new Error("this should not be called");
},
};
function parseAdditional(qualificationPrefix: string, text: string, textStartLine: number) {
const lines = text.split("\n"),
additional: Builder.AdditionalCommand[] = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.length > 2 && line.startsWith("| ") && line.endsWith(" |")) {
const keys = line
.slice(2, line.length - 2) // Remove start and end |.
.split(" | ") // Split into keys.
.map((k) => keyMapping[k.trim()]); // Normalize keys.
i++;
if (/^\|[-| ]+\|$/.test(lines[i])) {
i++;
}
while (i < lines.length) {
const line = lines[i];
if (!line.startsWith("| ") || !line.endsWith(" |")) {
break;
}
i++;
const obj: Builder.AdditionalCommand = { line: textStartLine + i },
values = line.slice(2, line.length - 2).split(" | ");
for (let j = 0; j < values.length; j++) {
const key = keys[j],
value = valueConverter[key](values[j].trim());
(obj as Record<string, any>)[key] = value;
}
if ("identifier" in obj) {
obj.qualifiedIdentifier = qualificationPrefix + obj.identifier;
}
additional.push(obj);
}
}
}
return additional;
}
/**
* Parses all the doc comments of functions in the given string of TypeScript
* code. Examples will be parsed using the given function.
*/
function parseDocComments(code: string, modulePath: string) {
let moduleDoc: string,
moduleDocStartLine: number,
moduleName: string;
const moduleHeaderMatch = moduleCommentRe.exec(code);
if (moduleHeaderMatch !== null) {
moduleDoc = moduleHeaderMatch[1].split("\n").map((line) => line.slice(3)).join("\n");
moduleDocStartLine = code.slice(0, moduleHeaderMatch.index).split("\n").length + 2;
moduleName = moduleHeaderMatch[2].replace(/^\.\//, "");
} else {
moduleDoc = "";
moduleDocStartLine = 0;
moduleName = path.basename(modulePath, ".ts");
}
if (verbose) {
console.log("Parsing doc comments in module", moduleName);
}
const modulePrefix = moduleName === "misc" ? "" : moduleName + ".",
functions: Builder.ParsedFunction[] = [];
for (let match = docCommentRe.exec(code); match !== null; match = docCommentRe.exec(code)) {
const indentationString = match[1],
docCommentString = match[2],
functionName = match[3],
parametersString = match[4],
returnTypeString = match[5],
startLine = countNewLines(code.slice(0, match.index)),
endLine = startLine + countNewLines(match[0]);
const indentation = indentationString.length,
returnType = returnTypeString.trim(),
parameters = parametersString
.split(/,(?![^:]+?[}>])/g)
.map((p) => p.trim())
.filter((p) => p.length > 0)
.map((p) => {
let match: RegExpExecArray | null;
if (match = /^(\w+\??|.+[}\]]): *(.+)$/s.exec(p)) {
return match.slice(1) as [string, string];
}
if (match = /^(\w+) *= *(\d+|true|false)$/.exec(p)) {
const type = match[2] === "true" || match[2] === "false"
? "Argument<boolean>"
: "number";
return [match[1], `${type} = ${match[2]}`] as [string, string];
}
if (match = /^(\w+) *= *(\w+)\.([\w.]+)$/.exec(p)) {
return [match[1], `${match[2]} = ${match[2]}.${match[3]}`] as [string, string];
}
if (match = /^(\.\.\.\w+): *(.+)$/.exec(p)) {
return [match[1], match[2]] as [string, string];
}
throw new Error(`unrecognized parameter pattern ${p}`);
}),
docComment = docCommentString
.split("\n")
.map((line) => line.slice(indentation).replace(/^ \* ?/g, ""))
.join("\n");
for (const parameter of parameters) {
if (parameter[0].endsWith("?")) {
// Optional parameters.
parameter[0] = parameter[0].slice(0, parameter[0].length - 1);
parameter[1] += " | undefined";
} else {
const match = /^(.+?)\s+=\s+(.+)$/.exec(parameter[1]);
if (match !== null) {
// Optional parameters with default values.
parameter[1] = match[1] + " | undefined";
}
}
}
const splitDocComment = docComment.split(/\n### Example\n/gm),
properties: Record<string, string> = {},
doc = splitDocComment[0].replace(/^@(param \w+|\w+)(?:\n| ((?:.+\n)(?: {2}.+\n)*))/gm,
(_, k: string, v?: string) => {
properties[k] = v?.replace(/\n {2}/g, " ").trim() ?? "";
return "";
}),
summary = /((?:.+(?:\n|$))+)/.exec(doc)![0].trim().replace(/\.$/, ""),
examplesStrings = splitDocComment.slice(1),
nameWithDot = functionName.replace(/_/g, ".");
if ("internal" in properties) {
continue;
}
let qualifiedName = modulePrefix;
if (nameWithDot === moduleName) {
qualifiedName = qualifiedName.replace(/\.$/, "");
} else {
qualifiedName += nameWithDot;
}
functions.push({
name: functionName,
nameWithDot,
qualifiedName,
startLine,
endLine,
doc,
properties,
summary,
examples: examplesStrings,
additional: parseAdditional(modulePrefix, splitDocComment[0], startLine),
parameters,
returnType: returnType.length === 0 ? undefined : returnType,
});
}
docCommentRe.lastIndex = 0;
return {
path: path.relative(path.dirname(__dirname), modulePath).replace(/\\/g, "/"),
name: moduleName,
doc: moduleDoc,
additional: parseAdditional(modulePrefix, moduleDoc, moduleDocStartLine),
functions,
functionNames: [...new Set(functions.map((f) => f.name))],
get commands() {
return getCommands(this);
},
get keybindings() {
return getKeybindings(this);
},
} as Builder.ParsedModule;
}
/**
* Mapping from character to corresponding VS Code keybinding.
*/
export const specialCharacterMapping = {
"~": "s-`",
"!": "s-1",
"@": "s-2",
"#": "s-3",
"$": "s-4",
"%": "s-5",
"^": "s-6",
"&": "s-7",
"*": "s-8",
"(": "s-9",
")": "s-0",
"_": "s--",
"+": "s-=",
"{": "s-[",
"}": "s-]",
"|": "s-\\",
":": "s-;",
'"': "s-'",
"<": "s-,",
">": "s-.",
"?": "s-/",
};
/**
* RegExp for keys of `specialCharacterMapping`.
*/
export const specialCharacterRegExp = /[~!@#$%^&*()+{}|:"<>?]|(?<!NumPad)_/g;
/**
* Async wrapper around the `glob` package.
*/
export function glob(pattern: string, options: { ignore?: string, cwd: string }) {
return new Promise<string[]>((resolve, reject) => {
G(pattern, options, (err, matches) => err ? reject(err) : resolve(matches));
});
}
/**
* A class used in .build.ts files.
*/
export class Builder {
private _apiModules?: Builder.ParsedModule[];
private _commandModules?: Builder.ParsedModule[];
private _beingBuilt = new Map<string, Promise<void>>();
/**
* Returns all modules for API files.
*/
public async getApiModules() {
if (this._apiModules !== undefined) {
return this._apiModules;
}
const apiFiles = await glob("src/api/**/*.ts", { cwd: __dirname, ignore: "**/*.build.ts" }),
apiModules = await Promise.all(
apiFiles.map(async (filepath) =>
parseDocComments(await fs.readFile(filepath, "utf-8"), filepath)));
return this._apiModules = apiModules.sort((a, b) => a.name.localeCompare(b.name));
}
/**
* Returns all modules for command files.
*/
public async getCommandModules() {
if (this._commandModules !== undefined) {
return this._commandModules;
}
const commandsGlob = `src/commands/**/*.ts`,
commandFiles = await glob(commandsGlob, { cwd: __dirname, ignore: "**/*.build.ts" }),
allCommandModules = await Promise.all(
commandFiles.map(async (filepath) =>
parseDocComments(await fs.readFile(filepath, "utf-8"), filepath))),
commandModules = allCommandModules.filter((m) => m.doc.length > 0);
return this._commandModules = commandModules.sort((a, b) => a.name.localeCompare(b.name));
}
/**
* Waits until the given files have been processed.
*/
public async waitFor(...files: readonly `${string}.build.ts`[]) {
await Promise.all(
files.map(async (file) => {
await this._beingBuilt.get(path.resolve(file));
}),
);
}
/**
* Updates all the given .build.ts files in parallel.
*/
public async buildFiles(filesToBuild: readonly string[], onError: (e: unknown) => void) {
await Promise.all(
filesToBuild.map(async (fileToBuild) => {
const absolutePath = path.resolve(fileToBuild),
promise = this._buildFile(fileToBuild)
.finally(() => this._beingBuilt.delete(absolutePath));
this._beingBuilt.set(absolutePath, promise);
await promise.catch(onError);
}),
);
}
/**
* Updates a .build.ts file.
*/
private async _buildFile(fileName: string) {
const relativeName = path.relative(__dirname, fileName),
relativeNameWithoutBuild = relativeName.replace(/build\.ts$/, ""),
modulePath = `./${relativeNameWithoutBuild}build`;
// Clear module cache if any.
delete require.cache[require.resolve(modulePath)];
const module: { build(builder: Builder): Promise<string> } = require(modulePath),
generatedContent = await module.build(this);
if (typeof generatedContent === "string") {
// Write result of `build` to the first file we find that has the same name
// as the build.ts file, but with any extension.
const prefix = path.basename(relativeNameWithoutBuild),
outputName = (await fs.readdir(path.dirname(fileName)))
.find((path) => path.startsWith(prefix) && !path.endsWith(".build.ts"))!,
outputPath = path.join(path.dirname(fileName), outputName),
outputContent = await fs.readFile(outputPath, "utf-8"),
outputContentHeader =
/^(?:[\s\S]+?\n)?.+Content below this line was auto-generated.+\n/m.exec(outputContent)![0];
await fs.writeFile(outputPath, outputContentHeader + generatedContent, "utf-8");
}
}
}
export declare namespace Builder {
export interface ParsedFunction {
readonly name: string;
readonly nameWithDot: string;
readonly qualifiedName: string;
readonly startLine: number;
readonly endLine: number;
readonly doc: string;
readonly properties: Record<string, string>;
readonly summary: string;
readonly examples: string[];
readonly additional: AdditionalCommand[];
readonly parameters: readonly [name: string, type: string][];
readonly returnType: string | undefined;
}
export interface AdditionalCommand {
title?: string;
identifier?: string;
qualifiedIdentifier?: string;
keys?: string;
commands?: string;
line: number;
}
export interface ParsedModule {
readonly path: string;
readonly name: string;
readonly doc: string;
readonly additional: readonly AdditionalCommand[];
readonly functions: readonly ParsedFunction[];
readonly functionNames: readonly string[];
readonly commands: {
readonly id: string;
readonly title: string;
readonly when?: string;
}[];
readonly keybindings: {
readonly title?: string;
readonly key: string;
readonly when: string;
readonly command: string;
readonly args?: any;
}[];
}
}
/**
* Parses the short "`s-a-b` (category: mode)"-like syntax for defining
* keybindings into a format compatible with VS Code keybindings.
*/
export function parseKeys(keys: string) {
if (keys.length === 0) {
return [];
}
return keys.replace("\n", ", ").split(/ *, (?=`)/g).map((keyString) => {
const [,, rawKeybinding, rawMetadata] = /^(`+)(.+?)\1 \((.+?)\)$/.exec(keyString)!,
keybinding = rawKeybinding.trim().replace(
specialCharacterRegExp, (m) => (specialCharacterMapping as Record<string, string>)[m]),
[, category, tags] = /(\w+): (.+)/.exec(rawMetadata)!;
// Reorder to match Ctrl+Shift+Alt+_
let key = "";
if (keybinding.includes("c-")) {
key += "Ctrl+";
}
if (keybinding.includes("s-")) {
key += "Shift+";
}
if (keybinding.includes("a-")) {
key += "Alt+";
}
const remainingKeybinding = keybinding.replace(/[csa]-/g, ""),
whenClauses = ["editorTextFocus"];
for (let tag of tags.split(", ")) {
const negate = tag.startsWith("!");
if (negate) {
tag = tag.slice(1);
}
switch (tag) {
case "normal":
case "insert":
case "visual":
case "input":
whenClauses.push(`dance.mode ${negate ? "!=" : "=="} '${tag}'`);
break;
case "recording":
whenClauses.push(`${negate ? "!" : ""}dance.isRecording`);
break;
case "prompt":
assert(!negate);
whenClauses.splice(whenClauses.indexOf("editorTextFocus"), 1);
whenClauses.push("inputFocus && !textInputFocus");
break;
default: {
const match = /^"(!?\w+)"$/.exec(tag);
if (match === null) {
throw new Error("unknown keybinding tag " + tag);
}
whenClauses.push((negate ? "!" : "") + match[1]);
break;
}
}
}
key += remainingKeybinding[0].toUpperCase() + remainingKeybinding.slice(1);
return {
category,
key,
when: whenClauses.join(" && "),
};
});
}
/**
* Returns all defined commands in the given module.
*/
function getCommands(module: Omit<Builder.ParsedModule, "commands">) {
// TODO: improve conditions
return [
...module.functions.map((f) => ({
id: `dance.${f.qualifiedName}`,
title: f.summary,
when: "dance.mode == 'normal'",
})),
...module.additional
.concat(...module.functions.flatMap((f) => f.additional))
.filter((a) => a.identifier !== undefined && a.title !== undefined)
.map((a) => ({
id: `dance.${a.qualifiedIdentifier}`,
title: a.title!,
when: "dance.mode == 'normal'",
})),
].sort((a, b) => a.id.localeCompare(b.id));
}
/**
* Returns all defined keybindings in the given module.
*/
function getKeybindings(module: Omit<Builder.ParsedModule, "keybindings">) {
return [
...module.functions.flatMap((f) => parseKeys(f.properties["keys"] ?? "").map((key) => ({
key: key.key,
when: key.when,
title: f.summary,
command: `dance.${f.qualifiedName}`,
}))),
...module.additional
.concat(...module.functions.flatMap((f) => f.additional))
.flatMap(({ title, keys, commands, qualifiedIdentifier }) => {
const parsedKeys = parseKeys(keys ?? "");
if (qualifiedIdentifier !== undefined) {
return parsedKeys.map((key) => ({
key: key.key,
when: key.when,
title,
command: `dance.${qualifiedIdentifier}`,
}));
}
const parsedCommands =
JSON.parse("[" + commands!.replace(/([$\w]+):/g, "\"$1\":") + "]") as any[];
if (parsedCommands.length === 1) {
let [command]: [string] = parsedCommands[0];
if (command[0] === ".") {
command = "dance" + command;
}
return parsedKeys.map((key) => ({
key: key.key,
when: key.when,
title,
command,
args: parsedCommands[0][1],
}));
}
return parsedKeys.map((key) => ({
key: key.key,
when: key.when,
title,
command: "dance.run",
args: {
commands: parsedCommands,
},
}));
}),
].sort((a, b) => a.command.localeCompare(b.command));
}
/**
* Given a multiline string, returns the same string with all lines starting
* with an indentation `>= by` reduced by `by` spaces.
*/
export function unindent(by: number): { (strings: TemplateStringsArray, ...args: any[]): string } {
const re = new RegExp(`^ {${by}}`, "gm");
return (strings: TemplateStringsArray, ...args: any[]) => {
const unindented = strings.map((s) => s.replace(re, ""));
return String.raw(Object.assign(unindented, { raw: unindented }), ...args)
.replace(/^ +$/gm, "");
};
}
/**
* The main entry point of the script.
*/
async function main() {
let success = true;
const ensureUpToDate = process.argv.includes("--ensure-up-to-date"),
check = process.argv.includes("--check"),
buildIndex = process.argv.indexOf("--build"),
build = buildIndex === -1 ? "**/*.build.ts" : process.argv[buildIndex + 1];
const contentsBefore: string[] = [],
fileNames = [
`${__dirname}/package.json`,
`${__dirname}/src/commands/README.md`,
`${__dirname}/src/commands/index.ts`,
];
if (ensureUpToDate) {
contentsBefore.push(...await Promise.all(fileNames.map((name) => fs.readFile(name, "utf-8"))));
}
const filesToBuild = await glob(build, { cwd: __dirname }),
builder = new Builder(),
buildErrors: unknown[] = [];
await builder.buildFiles(filesToBuild, (e) => buildErrors.push(e));
if (buildErrors.length > 0) {
console.error(buildErrors);
}
if (ensureUpToDate) {
const contentsAfter = await Promise.all(fileNames.map((name) => fs.readFile(name, "utf-8")));
for (let i = 0; i < fileNames.length; i++) {
if (verbose) {
console.log("Checking file", fileNames[i], "for diffs...");
}
// The built-in "assert" module displays a multiline diff if the strings
// are different, so we use it instead of comparing manually.
assert.strictEqual(contentsBefore[i], contentsAfter[i]);
}
}
if (check) {
const filesToCheck = await glob("src/commands/**/*.ts",
{ cwd: __dirname, ignore: "**/*.build.ts" }),
contentsToCheck = await Promise.all(filesToCheck.map((f) => fs.readFile(f, "utf-8")));
for (let i = 0; i < filesToCheck.length; i++) {
const fileToCheck = filesToCheck[i],
contentToCheck = contentsToCheck[i];
if (contentToCheck.includes("editor.selections")) {
console.error("File", fileToCheck, "includes forbidden access to editor.selections.");
success = false;
}
if (/^(export )?namespace/m.test(contentToCheck)) {
console.error("File", fileToCheck, "includes a non-`declare` namespace.");
success = false;
}
}
}
return success;
}
if (require.main === module) {
main().then(async (success) => {
if (!process.argv.includes("--watch")) {
process.exit(success ? 0 : 1);
}
const chokidar = await import("chokidar");
const watcher = chokidar.watch([
"**/*.build.ts",
"src/api/*.ts",
"src/commands/*.ts",
"test/suite/commands/*.md",
], {
ignored: "src/commands/load-all.ts",
});
let isGenerating = false;
watcher.on("change", async (path) => {
if (isGenerating) {
return;
}
console.log("Change detected at " + path + ", updating generated files...");
isGenerating = true;
try {
await main();
} finally {
isGenerating = false;
}
});
});
}