From 8b9c937f3008a8d89a3879d005e7df4867e03dbb Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:42:51 +0800 Subject: [PATCH] feat(i18n): add download scripts --- libs/datasource/i18n/package.json | 3 +- libs/datasource/i18n/src/index.ts | 19 +-- libs/datasource/i18n/src/scripts/api.ts | 83 +++++++++++- libs/datasource/i18n/src/scripts/download.ts | 134 +++++++++++++++++++ libs/datasource/i18n/src/scripts/request.ts | 1 + libs/datasource/i18n/src/scripts/sync.ts | 21 +-- libs/datasource/i18n/src/scripts/utils.ts | 3 + 7 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 libs/datasource/i18n/src/scripts/download.ts create mode 100644 libs/datasource/i18n/src/scripts/utils.ts diff --git a/libs/datasource/i18n/package.json b/libs/datasource/i18n/package.json index eb1657865e..25e7a82139 100644 --- a/libs/datasource/i18n/package.json +++ b/libs/datasource/i18n/package.json @@ -3,7 +3,8 @@ "version": "0.0.1", "scripts": { "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts", - "sync-languages:check": "pnpm run sync-languages --check" + "sync-languages:check": "pnpm run sync-languages --check", + "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts" }, "dependencies": { "i18next": "^21.9.1", diff --git a/libs/datasource/i18n/src/index.ts b/libs/datasource/i18n/src/index.ts index a421494bf8..f4428a9b09 100644 --- a/libs/datasource/i18n/src/index.ts +++ b/libs/datasource/i18n/src/index.ts @@ -4,8 +4,8 @@ import { initReactI18next, useTranslation, } from 'react-i18next'; -import en_US from './resources/en.json'; -import zh_CN from './resources/zh.json'; +import { LOCALES } from './resources'; +import type en_US from './resources/en.json'; // See https://react.i18next.com/latest/typescript declare module 'react-i18next' { @@ -21,22 +21,19 @@ declare module 'react-i18next' { const STORAGE_KEY = 'i18n_lng'; -const LOCALES = [ - { value: 'en', text: 'English', res: en_US }, - { value: 'zh', text: '简体中文', res: zh_CN }, -] as const; +export { i18n, useTranslation, I18nProvider, LOCALES }; const resources = LOCALES.reduce( - (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }), + (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }), {} ); -const fallbackLng = LOCALES[0].value; +const fallbackLng = LOCALES[0].tag; const standardizeLocale = (language: string) => { - if (LOCALES.find(locale => locale.value === language)) return language; + if (LOCALES.find(locale => locale.tag === language)) return language; if ( LOCALES.find( - locale => locale.value === language.slice(0, 2).toLowerCase() + locale => locale.tag === language.slice(0, 2).toLowerCase() ) ) return language; @@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => { }); const I18nProvider = I18nextProvider; - -export { i18n, useTranslation, I18nProvider, LOCALES }; diff --git a/libs/datasource/i18n/src/scripts/api.ts b/libs/datasource/i18n/src/scripts/api.ts index 52011e6042..b38ad5875b 100644 --- a/libs/datasource/i18n/src/scripts/api.ts +++ b/libs/datasource/i18n/src/scripts/api.ts @@ -5,15 +5,65 @@ import { fetchTolgee } from './request'; * Returns all project languages * * See https://tolgee.io/api#operation/getAll_6 + * @example + * ```ts + * const languages = [ + * { + * id: 1000016008, + * name: 'English', + * tag: 'en', + * originalName: 'English', + * flagEmoji: '🇬🇧', + * base: true + * }, + * { + * id: 1000016013, + * name: 'Spanish', + * tag: 'es', + * originalName: 'español', + * flagEmoji: '🇪🇸', + * base: false + * }, + * { + * id: 1000016009, + * name: 'Simplified Chinese', + * tag: 'zh-Hans', + * originalName: '简体中文', + * flagEmoji: '🇨🇳', + * base: false + * }, + * { + * id: 1000016012, + * name: 'Traditional Chinese', + * tag: 'zh-Hant', + * originalName: '繁體中文', + * flagEmoji: '🇭🇰', + * base: false + * } + * ] + * ``` */ -export const getAllProjectLanguages = async () => { - const url = '/languages?size=1000'; +export const getAllProjectLanguages = async (size = 1000) => { + const url = `/languages?size=${size}`; const resp = await fetchTolgee(url); if (resp.status < 200 || resp.status >= 300) { throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); } - const json = await resp.json(); - return json; + const json: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _embedded: { + languages: { + id: number; + name: string; + tag: string; + originalName: string; + flagEmoji: string; + base: boolean; + }[]; + }; + page: unknown; + } = await resp.json(); + return json._embedded.languages; }; /** @@ -48,6 +98,16 @@ export const getLanguagesTranslations = async ( return json; }; +export const getRemoteTranslations = async (languages: string) => { + const translations = await getLanguagesTranslations(languages); + if (!(languages in translations)) { + return {}; + } + // The assert is safe because we checked above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return translations[languages]!; +}; + /** * Creates new key * @@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => { // const keyId = // addTag(keyId, tag); }; + +/** + * Exports data + * + * See https://tolgee.io/api#operation/export_1 + */ +export const exportResources = async () => { + const url = `/export`; + const resp = await fetchTolgee(url); + + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + return resp; +}; diff --git a/libs/datasource/i18n/src/scripts/download.ts b/libs/datasource/i18n/src/scripts/download.ts new file mode 100644 index 0000000000..3b5cfc141e --- /dev/null +++ b/libs/datasource/i18n/src/scripts/download.ts @@ -0,0 +1,134 @@ +/* eslint-disable no-console */ +// cSpell:ignore Tolgee +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { format } from 'prettier'; +import { getAllProjectLanguages, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; + +const RES_DIR = path.resolve(process.cwd(), 'src', 'resources'); + +const countKeys = (obj: TranslationRes) => { + let count = 0; + Object.entries(obj).forEach(([key, value]) => { + if (typeof value === 'string') { + count++; + } else { + count += countKeys(value); + } + }); + return count; +}; + +const getBaseTranslations = async (baseLanguage: { tag: string }) => { + try { + const baseTranslationsStr = await fs.readFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + { encoding: 'utf8' } + ); + const baseTranslations = JSON.parse(baseTranslationsStr); + return baseTranslations; + } catch (e) { + console.error('base language:', JSON.stringify(baseLanguage)); + console.error('Failed to read base language', e); + const translations = await getRemoteTranslations(baseLanguage.tag); + await fs.writeFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + JSON.stringify(translations, null, 4) + ); + } +}; + +const main = async () => { + console.log('Loading project languages...'); + const languages = await getAllProjectLanguages(); + const baseLanguage = languages.find(language => language.base); + if (!baseLanguage) { + console.error(JSON.stringify(languages)); + throw new Error('Could not find base language'); + } + console.log( + `Loading ${baseLanguage.tag} languages translations as base...` + ); + + const baseTranslations = await getBaseTranslations(baseLanguage); + const baseKeyNum = countKeys(baseTranslations); + const languagesWithTranslations = await Promise.all( + languages.map(async language => { + console.log(`Loading ${language.tag} translations...`); + const translations = await getRemoteTranslations(language.tag); + const keyNum = countKeys(translations); + + return { + ...language, + translations, + completeRate: keyNum / baseKeyNum, + }; + }) + ); + + const availableLanguages = languagesWithTranslations.filter( + language => language.completeRate > 0 + ); + + availableLanguages + // skip base language + .filter(i => !i.base) + .forEach(async language => { + await fs.writeFile( + path.resolve(RES_DIR, `${language.tag}.json`), + JSON.stringify( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.': + '', + ...language.translations, + }, + null, + 4 + ) + ); + }); + + console.log('Generating meta data...'); + const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. + // Run \`pnpm run download-resources\` to regenerate. + // To overwrite this, please overwrite ${path.basename(__filename)} + ${availableLanguages + .map( + language => + `import ${language.tag.replaceAll('-', '_')} from './${ + language.tag + }.json'` + ) + .join('\n')} + + export const LOCALES = [ + ${availableLanguages + .map(({ translations, ...language }) => + JSON.stringify({ + ...language, + res: '__RES_PLACEHOLDER', + }).replace( + '"__RES_PLACEHOLDER"', + language.tag.replaceAll('-', '_') + ) + ) + .join(',\n')} + ] as const; + `; + + await fs.writeFile( + path.resolve(RES_DIR, 'index.ts'), + format(code, { + parser: 'typescript', + singleQuote: true, + trailingComma: 'es5', + tabWidth: 4, + arrowParens: 'avoid', + }) + ); + console.log('Done'); +}; + +main(); diff --git a/libs/datasource/i18n/src/scripts/request.ts b/libs/datasource/i18n/src/scripts/request.ts index c016909a24..b599c71957 100644 --- a/libs/datasource/i18n/src/scripts/request.ts +++ b/libs/datasource/i18n/src/scripts/request.ts @@ -46,6 +46,7 @@ const withTolgee = ( argArray[1].headers = headers; } } + // console.log('fetch', argArray); return target.apply(thisArg, argArray); }, }); diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts index a71a0de61f..10450eb3f6 100644 --- a/libs/datasource/i18n/src/scripts/sync.ts +++ b/libs/datasource/i18n/src/scripts/sync.ts @@ -1,30 +1,15 @@ +/* eslint-disable no-console */ // cSpell:ignore Tolgee import { readFile } from 'fs/promises'; import path from 'path'; -import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api'; +import { addTagByKey, createsNewKey, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json'); const BASE_LANGUAGES = 'en' as const; const DEPRECATED_TAG_NAME = 'unused' as const; -interface TranslationRes { - [x: string]: string | TranslationRes; -} - -const getRemoteTranslations = async (languages: string) => { - const translations = await getLanguagesTranslations(languages); - if (!(languages in translations)) { - console.log(translations); - throw new Error( - 'Failed to get base languages translation! base languages: ' + - languages - ); - } - // The assert is safe because we checked above - return translations[languages]!; -}; - /** * * @example diff --git a/libs/datasource/i18n/src/scripts/utils.ts b/libs/datasource/i18n/src/scripts/utils.ts new file mode 100644 index 0000000000..129a8febc3 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/utils.ts @@ -0,0 +1,3 @@ +export interface TranslationRes { + [x: string]: string | TranslationRes; +}