Merge pull request #397 from toeverything/feat/i18n-download

Feat/i18n download

.github/workflows/languages-download.yml

@@ -0,0 +1,84 @@
+name: Download Languages Resources
+
+on:
+    schedule:
+      - cron: "0 0 * * 5" # At 00:00(UTC) on Friday.
+    workflow_dispatch:
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+# See https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # The concurrency group contains the workflow name and the branch name for
+    # pull requests or the commit hash for any other events.
+    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+    cancel-in-progress: true
+
+jobs:
+    main:
+        strategy:
+            matrix:
+                node-version: [18]
+                os: [ubuntu-latest]
+        runs-on: ${{ matrix.os }}
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+
+            - name: Use pnpm
+              uses: pnpm/action-setup@v2
+              with:
+                  version: 7
+
+            - name: Use Node.js ${{ matrix.node-version }}
+              # https://github.com/actions/setup-node
+              uses: actions/setup-node@v3
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: 'pnpm'
+
+            - name: Install node modules
+              run: pnpm install
+
+            - name: Sync Languages
+              if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master'
+              working-directory: ./libs/datasource/i18n
+              run: pnpm run download-resources
+              env:
+                TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
+
+            - name: Push Branch
+              id: push
+              run: |
+                git add libs/datasource/i18n
+                # Do not proceed if there are no file differences
+                COMMIT=$(git rev-parse --verify origin/$TARGET_BRANCH || echo HEAD)
+                FILES_CHANGED=$(git diff-index --name-only --cached $COMMIT | wc -l)
+                if [[ "$FILES_CHANGED" = "0" ]]; then
+                  echo "No file changes detected."
+                  echo "::set-output name=skipPR::true"
+                  exit 0
+                fi
+                git config user.name 'github-actions[bot]'
+                git config user.email 'github-actions[bot]@users.noreply.github.com'
+                git commit --message 'feat(i18n): new translations'
+                git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY"
+                git push --force origin HEAD:$TARGET_BRANCH
+              env:
+                GITHUB_TOKEN: ${{ secrets.github_token }}
+                TARGET_BRANCH: bot/new-translations
+
+            - name: Get current date
+              id: date
+              run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+            # see https://github.com/repo-sync/pull-request
+            - name: Create Pull Request
+              if: steps.push.outputs.skipPR != 'true'
+              uses: repo-sync/pull-request@v2
+              with:
+                source_branch: 'bot/new-translations' # If blank, default: triggered branch
+                destination_branch: "develop"
+                pr_title: Update i18n (${{ steps.date.outputs.date }}) # Title of pull request
+                pr_label: 'data,bot' # Comma-separated list (no spaces)
+                github_token: ${{ secrets.GITHUB_TOKEN }}

libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx

@@ -53,10 +53,10 @@ export const SettingsList = () => {
                                 >
                                     {LOCALES.map(option => (
                                         <Option
-                                            key={option.value}
+                                            key={option.tag}
-                                            value={option.value}
+                                            value={option.tag}
                                         >
-                                            {option.text}
+                                            {option.originalName}
                                         </Option>
                                     ))}
                                 </Select>

libs/datasource/i18n/package.json

@@ -3,7 +3,8 @@
     "version": "0.0.1",
     "scripts": {
         "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts",
-        "sync-languages:check": "pnpm run sync-languages --check"
+        "sync-languages:check": "pnpm run sync-languages --check",
+        "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts"
     },
     "dependencies": {
         "i18next": "^21.9.1",

libs/datasource/i18n/src/base.json

@@ -1,24 +0,0 @@
-{
-    "Sync to Disk": "Sync to Disk",
-    "Share": "Share",
-    "WarningTips": {
-        "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
-        "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
-        "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
-    },
-    "Layout": "Layout",
-    "Comment": "Comment",
-    "Settings": "Settings",
-    "ComingSoon": "Layout Settings Coming Soon...",
-    "Duplicate Page": "Duplicate Page",
-    "Copy Page Link": "Copy Page Link",
-    "Language": "Language",
-    "Clear Workspace": "Clear Workspace",
-    "Export As Markdown": "Export As Markdown",
-    "Export As HTML": "Export As HTML",
-    "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
-    "Import Workspace": "Import Workspace",
-    "Export Workspace": "Export Workspace",
-    "Last edited by": "Last edited by {{name}}",
-    "Logout": "Logout"
-}

libs/datasource/i18n/src/index.ts

@@ -4,8 +4,8 @@ import {
     initReactI18next,
     useTranslation,
 } from 'react-i18next';
-import en_US from './resources/en.json';
+import { LOCALES } from './resources';
-import zh_CN from './resources/zh.json';
+import type en_US from './resources/en.json';
 
 // See https://react.i18next.com/latest/typescript
 declare module 'react-i18next' {
@@ -21,22 +21,19 @@ declare module 'react-i18next' {
 
 const STORAGE_KEY = 'i18n_lng';
 
-const LOCALES = [
+export { i18n, useTranslation, I18nProvider, LOCALES };
-    { value: 'en', text: 'English', res: en_US },
-    { value: 'zh', text: '简体中文', res: zh_CN },
-] as const;
 
 const resources = LOCALES.reduce<Resource>(
-    (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }),
+    (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }),
     {}
 );
 
-const fallbackLng = LOCALES[0].value;
+const fallbackLng = LOCALES[0].tag;
 const standardizeLocale = (language: string) => {
-    if (LOCALES.find(locale => locale.value === language)) return language;
+    if (LOCALES.find(locale => locale.tag === language)) return language;
     if (
         LOCALES.find(
-            locale => locale.value === language.slice(0, 2).toLowerCase()
+            locale => locale.tag === language.slice(0, 2).toLowerCase()
         )
     )
         return language;
@@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => {
 });
 
 const I18nProvider = I18nextProvider;
-
-export { i18n, useTranslation, I18nProvider, LOCALES };

libs/datasource/i18n/src/resources/index.ts

@@ -0,0 +1,28 @@
+// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+// Run `pnpm run download-resources` to regenerate.
+// To overwrite this, please overwrite download.ts
+import en from './en.json';
+import zh_Hans from './zh-Hans.json';
+
+export const LOCALES = [
+    {
+        id: 1000016008,
+        name: 'English',
+        tag: 'en',
+        originalName: 'English',
+        flagEmoji: '🇬🇧',
+        base: true,
+        completeRate: 1,
+        res: en,
+    },
+    {
+        id: 1000016009,
+        name: 'Simplified Chinese',
+        tag: 'zh-Hans',
+        originalName: '简体中文',
+        flagEmoji: '🇨🇳',
+        base: false,
+        completeRate: 1,
+        res: zh_Hans,
+    },
+] as const;

libs/datasource/i18n/src/resources/zh-Hans.json

@@ -1,4 +1,5 @@
 {
+    "// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.": "",
     "Sync to Disk": "同步到磁盘",
     "Share": "分享",
     "WarningTips": {

libs/datasource/i18n/src/scripts/api.ts

@@ -5,15 +5,65 @@ import { fetchTolgee } from './request';
  * Returns all project languages
  *
  * See https://tolgee.io/api#operation/getAll_6
+ * @example
+ * ```ts
+ * const languages = [
+ *   {
+ *     id: 1000016008,
+ *     name: 'English',
+ *     tag: 'en',
+ *     originalName: 'English',
+ *     flagEmoji: '🇬🇧',
+ *     base: true
+ *   },
+ *   {
+ *     id: 1000016013,
+ *     name: 'Spanish',
+ *     tag: 'es',
+ *     originalName: 'español',
+ *     flagEmoji: '🇪🇸',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016009,
+ *     name: 'Simplified Chinese',
+ *     tag: 'zh-Hans',
+ *     originalName: '简体中文',
+ *     flagEmoji: '🇨🇳',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016012,
+ *     name: 'Traditional Chinese',
+ *     tag: 'zh-Hant',
+ *     originalName: '繁體中文',
+ *     flagEmoji: '🇭🇰',
+ *     base: false
+ *   }
+ * ]
+ * ```
  */
-export const getAllProjectLanguages = async () => {
+export const getAllProjectLanguages = async (size = 1000) => {
-    const url = '/languages?size=1000';
+    const url = `/languages?size=${size}`;
     const resp = await fetchTolgee(url);
     if (resp.status < 200 || resp.status >= 300) {
         throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
     }
-    const json = await resp.json();
+    const json: {
-    return json;
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _embedded: {
+            languages: {
+                id: number;
+                name: string;
+                tag: string;
+                originalName: string;
+                flagEmoji: string;
+                base: boolean;
+            }[];
+        };
+        page: unknown;
+    } = await resp.json();
+    return json._embedded.languages;
 };
 
 /**
@@ -48,6 +98,16 @@ export const getLanguagesTranslations = async <T extends string>(
     return json;
 };
 
+export const getRemoteTranslations = async (languages: string) => {
+    const translations = await getLanguagesTranslations(languages);
+    if (!(languages in translations)) {
+        return {};
+    }
+    // The assert is safe because we checked above
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return translations[languages]!;
+};
+
 /**
  * Creates new key
  *
@@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => {
     // const keyId =
     // addTag(keyId, tag);
 };
+
+/**
+ * Exports data
+ *
+ * See https://tolgee.io/api#operation/export_1
+ */
+export const exportResources = async () => {
+    const url = `/export`;
+    const resp = await fetchTolgee(url);
+
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    return resp;
+};

libs/datasource/i18n/src/scripts/download.ts

@@ -0,0 +1,134 @@
+/* eslint-disable no-console */
+// cSpell:ignore Tolgee
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { format } from 'prettier';
+import { getAllProjectLanguages, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
+
+const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
+
+const countKeys = (obj: TranslationRes) => {
+    let count = 0;
+    Object.entries(obj).forEach(([key, value]) => {
+        if (typeof value === 'string') {
+            count++;
+        } else {
+            count += countKeys(value);
+        }
+    });
+    return count;
+};
+
+const getBaseTranslations = async (baseLanguage: { tag: string }) => {
+    try {
+        const baseTranslationsStr = await fs.readFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            { encoding: 'utf8' }
+        );
+        const baseTranslations = JSON.parse(baseTranslationsStr);
+        return baseTranslations;
+    } catch (e) {
+        console.error('base language:', JSON.stringify(baseLanguage));
+        console.error('Failed to read base language', e);
+        const translations = await getRemoteTranslations(baseLanguage.tag);
+        await fs.writeFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            JSON.stringify(translations, null, 4)
+        );
+    }
+};
+
+const main = async () => {
+    console.log('Loading project languages...');
+    const languages = await getAllProjectLanguages();
+    const baseLanguage = languages.find(language => language.base);
+    if (!baseLanguage) {
+        console.error(JSON.stringify(languages));
+        throw new Error('Could not find base language');
+    }
+    console.log(
+        `Loading ${baseLanguage.tag} languages translations as base...`
+    );
+
+    const baseTranslations = await getBaseTranslations(baseLanguage);
+    const baseKeyNum = countKeys(baseTranslations);
+    const languagesWithTranslations = await Promise.all(
+        languages.map(async language => {
+            console.log(`Loading ${language.tag} translations...`);
+            const translations = await getRemoteTranslations(language.tag);
+            const keyNum = countKeys(translations);
+
+            return {
+                ...language,
+                translations,
+                completeRate: keyNum / baseKeyNum,
+            };
+        })
+    );
+
+    const availableLanguages = languagesWithTranslations.filter(
+        language => language.completeRate > 0
+    );
+
+    availableLanguages
+        // skip base language
+        .filter(i => !i.base)
+        .forEach(async language => {
+            await fs.writeFile(
+                path.resolve(RES_DIR, `${language.tag}.json`),
+                JSON.stringify(
+                    {
+                        // eslint-disable-next-line @typescript-eslint/naming-convention
+                        '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
+                            '',
+                        ...language.translations,
+                    },
+                    null,
+                    4
+                ) + '\n'
+            );
+        });
+
+    console.log('Generating meta data...');
+    const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+        // Run \`pnpm run download-resources\` to regenerate.
+        // To overwrite this, please overwrite ${path.basename(__filename)}
+        ${availableLanguages
+            .map(
+                language =>
+                    `import ${language.tag.replaceAll('-', '_')} from './${
+                        language.tag
+                    }.json'`
+            )
+            .join('\n')}
+
+        export const LOCALES = [
+            ${availableLanguages
+                .map(({ translations, ...language }) =>
+                    JSON.stringify({
+                        ...language,
+                        res: '__RES_PLACEHOLDER',
+                    }).replace(
+                        '"__RES_PLACEHOLDER"',
+                        language.tag.replaceAll('-', '_')
+                    )
+                )
+                .join(',\n')}
+        ] as const;
+        `;
+
+    await fs.writeFile(
+        path.resolve(RES_DIR, 'index.ts'),
+        format(code, {
+            parser: 'typescript',
+            singleQuote: true,
+            trailingComma: 'es5',
+            tabWidth: 4,
+            arrowParens: 'avoid',
+        })
+    );
+    console.log('Done');
+};
+
+main();

libs/datasource/i18n/src/scripts/request.ts

@@ -46,6 +46,7 @@ const withTolgee = (
                     argArray[1].headers = headers;
                 }
             }
+            // console.log('fetch', argArray);
             return target.apply(thisArg, argArray);
         },
     });

libs/datasource/i18n/src/scripts/sync.ts

@@ -1,30 +1,20 @@
+/* eslint-disable no-console */
 // cSpell:ignore Tolgee
 import { readFile } from 'fs/promises';
 import path from 'path';
-import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+import { addTagByKey, createsNewKey, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
 
-const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
+const BASE_JSON_PATH = path.resolve(
+    process.cwd(),
+    'src',
+    'resources',
+    'en.json'
+);
 const BASE_LANGUAGES = 'en' as const;
 
 const DEPRECATED_TAG_NAME = 'unused' as const;
 
-interface TranslationRes {
-    [x: string]: string | TranslationRes;
-}
-
-const getRemoteTranslations = async (languages: string) => {
-    const translations = await getLanguagesTranslations(languages);
-    if (!(languages in translations)) {
-        console.log(translations);
-        throw new Error(
-            'Failed to get base languages translation! base languages: ' +
-                languages
-        );
-    }
-    // The assert is safe because we checked above
-    return translations[languages]!;
-};
-
 /**
  *
  * @example

libs/datasource/i18n/src/scripts/utils.ts

@@ -0,0 +1,3 @@
+export interface TranslationRes {
+    [x: string]: string | TranslationRes;
+}