From 8b9c937f3008a8d89a3879d005e7df4867e03dbb Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:42:51 +0800 Subject: [PATCH 1/5] feat(i18n): add download scripts --- libs/datasource/i18n/package.json | 3 +- libs/datasource/i18n/src/index.ts | 19 +-- libs/datasource/i18n/src/scripts/api.ts | 83 +++++++++++- libs/datasource/i18n/src/scripts/download.ts | 134 +++++++++++++++++++ libs/datasource/i18n/src/scripts/request.ts | 1 + libs/datasource/i18n/src/scripts/sync.ts | 21 +-- libs/datasource/i18n/src/scripts/utils.ts | 3 + 7 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 libs/datasource/i18n/src/scripts/download.ts create mode 100644 libs/datasource/i18n/src/scripts/utils.ts diff --git a/libs/datasource/i18n/package.json b/libs/datasource/i18n/package.json index eb1657865e..25e7a82139 100644 --- a/libs/datasource/i18n/package.json +++ b/libs/datasource/i18n/package.json @@ -3,7 +3,8 @@ "version": "0.0.1", "scripts": { "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts", - "sync-languages:check": "pnpm run sync-languages --check" + "sync-languages:check": "pnpm run sync-languages --check", + "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts" }, "dependencies": { "i18next": "^21.9.1", diff --git a/libs/datasource/i18n/src/index.ts b/libs/datasource/i18n/src/index.ts index a421494bf8..f4428a9b09 100644 --- a/libs/datasource/i18n/src/index.ts +++ b/libs/datasource/i18n/src/index.ts @@ -4,8 +4,8 @@ import { initReactI18next, useTranslation, } from 'react-i18next'; -import en_US from './resources/en.json'; -import zh_CN from './resources/zh.json'; +import { LOCALES } from './resources'; +import type en_US from './resources/en.json'; // See https://react.i18next.com/latest/typescript declare module 'react-i18next' { @@ -21,22 +21,19 @@ declare module 'react-i18next' { const STORAGE_KEY = 'i18n_lng'; -const LOCALES = [ - { value: 'en', text: 'English', res: en_US }, - { value: 'zh', text: '简体中文', res: zh_CN }, -] as const; +export { i18n, useTranslation, I18nProvider, LOCALES }; const resources = LOCALES.reduce( - (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }), + (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }), {} ); -const fallbackLng = LOCALES[0].value; +const fallbackLng = LOCALES[0].tag; const standardizeLocale = (language: string) => { - if (LOCALES.find(locale => locale.value === language)) return language; + if (LOCALES.find(locale => locale.tag === language)) return language; if ( LOCALES.find( - locale => locale.value === language.slice(0, 2).toLowerCase() + locale => locale.tag === language.slice(0, 2).toLowerCase() ) ) return language; @@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => { }); const I18nProvider = I18nextProvider; - -export { i18n, useTranslation, I18nProvider, LOCALES }; diff --git a/libs/datasource/i18n/src/scripts/api.ts b/libs/datasource/i18n/src/scripts/api.ts index 52011e6042..b38ad5875b 100644 --- a/libs/datasource/i18n/src/scripts/api.ts +++ b/libs/datasource/i18n/src/scripts/api.ts @@ -5,15 +5,65 @@ import { fetchTolgee } from './request'; * Returns all project languages * * See https://tolgee.io/api#operation/getAll_6 + * @example + * ```ts + * const languages = [ + * { + * id: 1000016008, + * name: 'English', + * tag: 'en', + * originalName: 'English', + * flagEmoji: '🇬🇧', + * base: true + * }, + * { + * id: 1000016013, + * name: 'Spanish', + * tag: 'es', + * originalName: 'español', + * flagEmoji: '🇪🇸', + * base: false + * }, + * { + * id: 1000016009, + * name: 'Simplified Chinese', + * tag: 'zh-Hans', + * originalName: '简体中文', + * flagEmoji: '🇨🇳', + * base: false + * }, + * { + * id: 1000016012, + * name: 'Traditional Chinese', + * tag: 'zh-Hant', + * originalName: '繁體中文', + * flagEmoji: '🇭🇰', + * base: false + * } + * ] + * ``` */ -export const getAllProjectLanguages = async () => { - const url = '/languages?size=1000'; +export const getAllProjectLanguages = async (size = 1000) => { + const url = `/languages?size=${size}`; const resp = await fetchTolgee(url); if (resp.status < 200 || resp.status >= 300) { throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); } - const json = await resp.json(); - return json; + const json: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _embedded: { + languages: { + id: number; + name: string; + tag: string; + originalName: string; + flagEmoji: string; + base: boolean; + }[]; + }; + page: unknown; + } = await resp.json(); + return json._embedded.languages; }; /** @@ -48,6 +98,16 @@ export const getLanguagesTranslations = async ( return json; }; +export const getRemoteTranslations = async (languages: string) => { + const translations = await getLanguagesTranslations(languages); + if (!(languages in translations)) { + return {}; + } + // The assert is safe because we checked above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return translations[languages]!; +}; + /** * Creates new key * @@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => { // const keyId = // addTag(keyId, tag); }; + +/** + * Exports data + * + * See https://tolgee.io/api#operation/export_1 + */ +export const exportResources = async () => { + const url = `/export`; + const resp = await fetchTolgee(url); + + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + return resp; +}; diff --git a/libs/datasource/i18n/src/scripts/download.ts b/libs/datasource/i18n/src/scripts/download.ts new file mode 100644 index 0000000000..3b5cfc141e --- /dev/null +++ b/libs/datasource/i18n/src/scripts/download.ts @@ -0,0 +1,134 @@ +/* eslint-disable no-console */ +// cSpell:ignore Tolgee +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { format } from 'prettier'; +import { getAllProjectLanguages, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; + +const RES_DIR = path.resolve(process.cwd(), 'src', 'resources'); + +const countKeys = (obj: TranslationRes) => { + let count = 0; + Object.entries(obj).forEach(([key, value]) => { + if (typeof value === 'string') { + count++; + } else { + count += countKeys(value); + } + }); + return count; +}; + +const getBaseTranslations = async (baseLanguage: { tag: string }) => { + try { + const baseTranslationsStr = await fs.readFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + { encoding: 'utf8' } + ); + const baseTranslations = JSON.parse(baseTranslationsStr); + return baseTranslations; + } catch (e) { + console.error('base language:', JSON.stringify(baseLanguage)); + console.error('Failed to read base language', e); + const translations = await getRemoteTranslations(baseLanguage.tag); + await fs.writeFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + JSON.stringify(translations, null, 4) + ); + } +}; + +const main = async () => { + console.log('Loading project languages...'); + const languages = await getAllProjectLanguages(); + const baseLanguage = languages.find(language => language.base); + if (!baseLanguage) { + console.error(JSON.stringify(languages)); + throw new Error('Could not find base language'); + } + console.log( + `Loading ${baseLanguage.tag} languages translations as base...` + ); + + const baseTranslations = await getBaseTranslations(baseLanguage); + const baseKeyNum = countKeys(baseTranslations); + const languagesWithTranslations = await Promise.all( + languages.map(async language => { + console.log(`Loading ${language.tag} translations...`); + const translations = await getRemoteTranslations(language.tag); + const keyNum = countKeys(translations); + + return { + ...language, + translations, + completeRate: keyNum / baseKeyNum, + }; + }) + ); + + const availableLanguages = languagesWithTranslations.filter( + language => language.completeRate > 0 + ); + + availableLanguages + // skip base language + .filter(i => !i.base) + .forEach(async language => { + await fs.writeFile( + path.resolve(RES_DIR, `${language.tag}.json`), + JSON.stringify( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.': + '', + ...language.translations, + }, + null, + 4 + ) + ); + }); + + console.log('Generating meta data...'); + const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. + // Run \`pnpm run download-resources\` to regenerate. + // To overwrite this, please overwrite ${path.basename(__filename)} + ${availableLanguages + .map( + language => + `import ${language.tag.replaceAll('-', '_')} from './${ + language.tag + }.json'` + ) + .join('\n')} + + export const LOCALES = [ + ${availableLanguages + .map(({ translations, ...language }) => + JSON.stringify({ + ...language, + res: '__RES_PLACEHOLDER', + }).replace( + '"__RES_PLACEHOLDER"', + language.tag.replaceAll('-', '_') + ) + ) + .join(',\n')} + ] as const; + `; + + await fs.writeFile( + path.resolve(RES_DIR, 'index.ts'), + format(code, { + parser: 'typescript', + singleQuote: true, + trailingComma: 'es5', + tabWidth: 4, + arrowParens: 'avoid', + }) + ); + console.log('Done'); +}; + +main(); diff --git a/libs/datasource/i18n/src/scripts/request.ts b/libs/datasource/i18n/src/scripts/request.ts index c016909a24..b599c71957 100644 --- a/libs/datasource/i18n/src/scripts/request.ts +++ b/libs/datasource/i18n/src/scripts/request.ts @@ -46,6 +46,7 @@ const withTolgee = ( argArray[1].headers = headers; } } + // console.log('fetch', argArray); return target.apply(thisArg, argArray); }, }); diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts index a71a0de61f..10450eb3f6 100644 --- a/libs/datasource/i18n/src/scripts/sync.ts +++ b/libs/datasource/i18n/src/scripts/sync.ts @@ -1,30 +1,15 @@ +/* eslint-disable no-console */ // cSpell:ignore Tolgee import { readFile } from 'fs/promises'; import path from 'path'; -import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api'; +import { addTagByKey, createsNewKey, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json'); const BASE_LANGUAGES = 'en' as const; const DEPRECATED_TAG_NAME = 'unused' as const; -interface TranslationRes { - [x: string]: string | TranslationRes; -} - -const getRemoteTranslations = async (languages: string) => { - const translations = await getLanguagesTranslations(languages); - if (!(languages in translations)) { - console.log(translations); - throw new Error( - 'Failed to get base languages translation! base languages: ' + - languages - ); - } - // The assert is safe because we checked above - return translations[languages]!; -}; - /** * * @example diff --git a/libs/datasource/i18n/src/scripts/utils.ts b/libs/datasource/i18n/src/scripts/utils.ts new file mode 100644 index 0000000000..129a8febc3 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/utils.ts @@ -0,0 +1,3 @@ +export interface TranslationRes { + [x: string]: string | TranslationRes; +} From 08d331e56e057b8783b4f1fdcb0d1f938b08106d Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:43:11 +0800 Subject: [PATCH 2/5] feat(i18n): add codegen --- libs/datasource/i18n/src/base.json | 24 ---------------- libs/datasource/i18n/src/resources/index.ts | 28 +++++++++++++++++++ .../src/resources/{zh.json => zh-Hans.json} | 1 + 3 files changed, 29 insertions(+), 24 deletions(-) delete mode 100644 libs/datasource/i18n/src/base.json create mode 100644 libs/datasource/i18n/src/resources/index.ts rename libs/datasource/i18n/src/resources/{zh.json => zh-Hans.json} (93%) diff --git a/libs/datasource/i18n/src/base.json b/libs/datasource/i18n/src/base.json deleted file mode 100644 index e243d71db1..0000000000 --- a/libs/datasource/i18n/src/base.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "Sync to Disk": "Sync to Disk", - "Share": "Share", - "WarningTips": { - "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge", - "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.", - "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data" - }, - "Layout": "Layout", - "Comment": "Comment", - "Settings": "Settings", - "ComingSoon": "Layout Settings Coming Soon...", - "Duplicate Page": "Duplicate Page", - "Copy Page Link": "Copy Page Link", - "Language": "Language", - "Clear Workspace": "Clear Workspace", - "Export As Markdown": "Export As Markdown", - "Export As HTML": "Export As HTML", - "Export As PDF (Unsupported)": "Export As PDF (Unsupported)", - "Import Workspace": "Import Workspace", - "Export Workspace": "Export Workspace", - "Last edited by": "Last edited by {{name}}", - "Logout": "Logout" -} diff --git a/libs/datasource/i18n/src/resources/index.ts b/libs/datasource/i18n/src/resources/index.ts new file mode 100644 index 0000000000..1d3e875c60 --- /dev/null +++ b/libs/datasource/i18n/src/resources/index.ts @@ -0,0 +1,28 @@ +// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +// Run `pnpm run download-resources` to regenerate. +// To overwrite this, please overwrite download.ts +import en from './en.json'; +import zh_Hans from './zh-Hans.json'; + +export const LOCALES = [ + { + id: 1000016008, + name: 'English', + tag: 'en', + originalName: 'English', + flagEmoji: '🇬🇧', + base: true, + completeRate: 1, + res: en, + }, + { + id: 1000016009, + name: 'Simplified Chinese', + tag: 'zh-Hans', + originalName: '简体中文', + flagEmoji: '🇨🇳', + base: false, + completeRate: 1, + res: zh_Hans, + }, +] as const; diff --git a/libs/datasource/i18n/src/resources/zh.json b/libs/datasource/i18n/src/resources/zh-Hans.json similarity index 93% rename from libs/datasource/i18n/src/resources/zh.json rename to libs/datasource/i18n/src/resources/zh-Hans.json index fe72ae542d..5122e4b881 100644 --- a/libs/datasource/i18n/src/resources/zh.json +++ b/libs/datasource/i18n/src/resources/zh-Hans.json @@ -1,4 +1,5 @@ { + "// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.": "", "Sync to Disk": "同步到磁盘", "Share": "分享", "WarningTips": { From b54f2d04514d8e619393cc692a37e6cc519f63a1 Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:43:26 +0800 Subject: [PATCH 3/5] chore: clean --- .../layout/src/settings-sidebar/Settings/SettingsList.tsx | 6 +++--- libs/datasource/i18n/src/scripts/download.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx index a308614d13..8ebaf90a82 100644 --- a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx +++ b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx @@ -53,10 +53,10 @@ export const SettingsList = () => { > {LOCALES.map(option => ( ))} diff --git a/libs/datasource/i18n/src/scripts/download.ts b/libs/datasource/i18n/src/scripts/download.ts index 3b5cfc141e..f45261c258 100644 --- a/libs/datasource/i18n/src/scripts/download.ts +++ b/libs/datasource/i18n/src/scripts/download.ts @@ -86,7 +86,7 @@ const main = async () => { }, null, 4 - ) + ) + '\n' ); }); From e7ca1a7a2579499b492eee302ae971177670f744 Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Sat, 10 Sep 2022 02:32:28 +0800 Subject: [PATCH 4/5] fix(i18n): base json path --- libs/datasource/i18n/src/scripts/sync.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts index 10450eb3f6..c2ba692dc5 100644 --- a/libs/datasource/i18n/src/scripts/sync.ts +++ b/libs/datasource/i18n/src/scripts/sync.ts @@ -5,7 +5,12 @@ import path from 'path'; import { addTagByKey, createsNewKey, getRemoteTranslations } from './api'; import type { TranslationRes } from './utils'; -const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json'); +const BASE_JSON_PATH = path.resolve( + process.cwd(), + 'src', + 'resources', + 'en.json' +); const BASE_LANGUAGES = 'en' as const; const DEPRECATED_TAG_NAME = 'unused' as const; From 02be48827158c0dbcfd0f76756b4ed99a0ddb86a Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Sat, 10 Sep 2022 02:47:18 +0800 Subject: [PATCH 5/5] feat(i18n): add auto update ci --- .github/workflows/languages-download.yml | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/languages-download.yml diff --git a/.github/workflows/languages-download.yml b/.github/workflows/languages-download.yml new file mode 100644 index 0000000000..4e180bccce --- /dev/null +++ b/.github/workflows/languages-download.yml @@ -0,0 +1,84 @@ +name: Download Languages Resources + +on: + schedule: + - cron: "0 0 * * 5" # At 00:00(UTC) on Friday. + workflow_dispatch: + +# Cancels all previous workflow runs for pull requests that have not completed. +# See https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # The concurrency group contains the workflow name and the branch name for + # pull requests or the commit hash for any other events. + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + main: + strategy: + matrix: + node-version: [18] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Use pnpm + uses: pnpm/action-setup@v2 + with: + version: 7 + + - name: Use Node.js ${{ matrix.node-version }} + # https://github.com/actions/setup-node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'pnpm' + + - name: Install node modules + run: pnpm install + + - name: Sync Languages + if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' + working-directory: ./libs/datasource/i18n + run: pnpm run download-resources + env: + TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }} + + - name: Push Branch + id: push + run: | + git add libs/datasource/i18n + # Do not proceed if there are no file differences + COMMIT=$(git rev-parse --verify origin/$TARGET_BRANCH || echo HEAD) + FILES_CHANGED=$(git diff-index --name-only --cached $COMMIT | wc -l) + if [[ "$FILES_CHANGED" = "0" ]]; then + echo "No file changes detected." + echo "::set-output name=skipPR::true" + exit 0 + fi + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit --message 'feat(i18n): new translations' + git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY" + git push --force origin HEAD:$TARGET_BRANCH + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + TARGET_BRANCH: bot/new-translations + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + # see https://github.com/repo-sync/pull-request + - name: Create Pull Request + if: steps.push.outputs.skipPR != 'true' + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/new-translations' # If blank, default: triggered branch + destination_branch: "develop" + pr_title: Update i18n (${{ steps.date.outputs.date }}) # Title of pull request + pr_label: 'data,bot' # Comma-separated list (no spaces) + github_token: ${{ secrets.GITHUB_TOKEN }}