mirror of
https://github.com/swc-project/swc.git
synced 2024-12-23 05:32:09 +03:00
80 lines
1.8 KiB
TypeScript
80 lines
1.8 KiB
TypeScript
// Loaded from https://deno.land/std/datetime/tokenizer.ts
|
|
|
|
|
|
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
|
|
|
|
export type Token = {
|
|
type: string;
|
|
value: string | number;
|
|
index: number;
|
|
[key: string]: unknown;
|
|
};
|
|
|
|
export interface ReceiverResult {
|
|
[name: string]: string | number | unknown;
|
|
}
|
|
export type CallbackResult = {
|
|
type: string;
|
|
value: string | number;
|
|
[key: string]: unknown;
|
|
};
|
|
type CallbackFunction = (value: unknown) => CallbackResult;
|
|
|
|
export type TestResult = { value: unknown; length: number } | undefined;
|
|
export type TestFunction = (
|
|
string: string,
|
|
) => TestResult | undefined;
|
|
|
|
export interface Rule {
|
|
test: TestFunction;
|
|
fn: CallbackFunction;
|
|
}
|
|
|
|
export class Tokenizer {
|
|
rules: Rule[];
|
|
|
|
constructor(rules: Rule[] = []) {
|
|
this.rules = rules;
|
|
}
|
|
|
|
addRule(test: TestFunction, fn: CallbackFunction): Tokenizer {
|
|
this.rules.push({ test, fn });
|
|
return this;
|
|
}
|
|
|
|
tokenize(
|
|
string: string,
|
|
receiver = (token: Token): ReceiverResult => token,
|
|
): ReceiverResult[] {
|
|
function* generator(rules: Rule[]): IterableIterator<ReceiverResult> {
|
|
let index = 0;
|
|
for (const rule of rules) {
|
|
const result = rule.test(string);
|
|
if (result) {
|
|
const { value, length } = result;
|
|
index += length;
|
|
string = string.slice(length);
|
|
const token = { ...rule.fn(value), index };
|
|
yield receiver(token);
|
|
yield* generator(rules);
|
|
}
|
|
}
|
|
}
|
|
const tokenGenerator = generator(this.rules);
|
|
|
|
const tokens: ReceiverResult[] = [];
|
|
|
|
for (const token of tokenGenerator) {
|
|
tokens.push(token);
|
|
}
|
|
|
|
if (string.length) {
|
|
throw new Error(
|
|
`parser error: string not fully parsed! ${string.slice(0, 25)}`,
|
|
);
|
|
}
|
|
|
|
return tokens;
|
|
}
|
|
}
|