AFFiNE/packages/backend/native/index.d.ts
Brooooooklyn c7ddd679fd
feat(server): use native tokenizer impl (#6960)
### Benchmark

`yarn workspace @affine/server-native bench`

```
┌─────────┬────────────┬─────────┬────────────────────┬──────────┬─────────┐
│ (index) │ Task Name  │ ops/sec │ Average Time (ns)  │ Margin   │ Samples │
├─────────┼────────────┼─────────┼────────────────────┼──────────┼─────────┤
│ 0       │ 'tiktoken' │ '5'     │ 176932518.76000002 │ '±4.71%' │ 100     │
│ 1       │ 'native'   │ '16'    │ 61041597.51000003  │ '±0.60%' │ 100     │
└─────────┴────────────┴─────────┴────────────────────┴──────────┴─────────┘
```
2024-05-16 07:55:10 +00:00

21 lines
678 B
TypeScript

/* auto-generated by NAPI-RS */
/* eslint-disable */
export class Tokenizer {
count(content: string, allowedSpecial?: Array<string> | undefined | null): number
}
export function fromModelName(modelName: string): Tokenizer | null
export function getMime(input: Uint8Array): string
/**
* Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
* result binary.
*/
export function mergeUpdatesInApplyWay(updates: Array<Buffer>): Buffer
export function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
export function verifyChallengeResponse(response: string, bits: number, resource: string): Promise<boolean>