feat(i18n): add download scripts

libs/datasource/i18n/package.json

@@ -3,7 +3,8 @@
     "version": "0.0.1",
     "scripts": {
         "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts",
-        "sync-languages:check": "pnpm run sync-languages --check"
+        "sync-languages:check": "pnpm run sync-languages --check",
+        "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts"
     },
     "dependencies": {
         "i18next": "^21.9.1",

libs/datasource/i18n/src/index.ts

@@ -4,8 +4,8 @@ import {
     initReactI18next,
     useTranslation,
 } from 'react-i18next';
-import en_US from './resources/en.json';
+import { LOCALES } from './resources';
-import zh_CN from './resources/zh.json';
+import type en_US from './resources/en.json';
 
 // See https://react.i18next.com/latest/typescript
 declare module 'react-i18next' {
@@ -21,22 +21,19 @@ declare module 'react-i18next' {
 
 const STORAGE_KEY = 'i18n_lng';
 
-const LOCALES = [
+export { i18n, useTranslation, I18nProvider, LOCALES };
-    { value: 'en', text: 'English', res: en_US },
-    { value: 'zh', text: '简体中文', res: zh_CN },
-] as const;
 
 const resources = LOCALES.reduce<Resource>(
-    (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }),
+    (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }),
     {}
 );
 
-const fallbackLng = LOCALES[0].value;
+const fallbackLng = LOCALES[0].tag;
 const standardizeLocale = (language: string) => {
-    if (LOCALES.find(locale => locale.value === language)) return language;
+    if (LOCALES.find(locale => locale.tag === language)) return language;
     if (
         LOCALES.find(
-            locale => locale.value === language.slice(0, 2).toLowerCase()
+            locale => locale.tag === language.slice(0, 2).toLowerCase()
         )
     )
         return language;
@@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => {
 });
 
 const I18nProvider = I18nextProvider;
-
-export { i18n, useTranslation, I18nProvider, LOCALES };

libs/datasource/i18n/src/scripts/api.ts

@@ -5,15 +5,65 @@ import { fetchTolgee } from './request';
  * Returns all project languages
  *
  * See https://tolgee.io/api#operation/getAll_6
+ * @example
+ * ```ts
+ * const languages = [
+ *   {
+ *     id: 1000016008,
+ *     name: 'English',
+ *     tag: 'en',
+ *     originalName: 'English',
+ *     flagEmoji: '🇬🇧',
+ *     base: true
+ *   },
+ *   {
+ *     id: 1000016013,
+ *     name: 'Spanish',
+ *     tag: 'es',
+ *     originalName: 'español',
+ *     flagEmoji: '🇪🇸',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016009,
+ *     name: 'Simplified Chinese',
+ *     tag: 'zh-Hans',
+ *     originalName: '简体中文',
+ *     flagEmoji: '🇨🇳',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016012,
+ *     name: 'Traditional Chinese',
+ *     tag: 'zh-Hant',
+ *     originalName: '繁體中文',
+ *     flagEmoji: '🇭🇰',
+ *     base: false
+ *   }
+ * ]
+ * ```
  */
-export const getAllProjectLanguages = async () => {
+export const getAllProjectLanguages = async (size = 1000) => {
-    const url = '/languages?size=1000';
+    const url = `/languages?size=${size}`;
     const resp = await fetchTolgee(url);
     if (resp.status < 200 || resp.status >= 300) {
         throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
     }
-    const json = await resp.json();
+    const json: {
-    return json;
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _embedded: {
+            languages: {
+                id: number;
+                name: string;
+                tag: string;
+                originalName: string;
+                flagEmoji: string;
+                base: boolean;
+            }[];
+        };
+        page: unknown;
+    } = await resp.json();
+    return json._embedded.languages;
 };
 
 /**
@@ -48,6 +98,16 @@ export const getLanguagesTranslations = async <T extends string>(
     return json;
 };
 
+export const getRemoteTranslations = async (languages: string) => {
+    const translations = await getLanguagesTranslations(languages);
+    if (!(languages in translations)) {
+        return {};
+    }
+    // The assert is safe because we checked above
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return translations[languages]!;
+};
+
 /**
  * Creates new key
  *
@@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => {
     // const keyId =
     // addTag(keyId, tag);
 };
+
+/**
+ * Exports data
+ *
+ * See https://tolgee.io/api#operation/export_1
+ */
+export const exportResources = async () => {
+    const url = `/export`;
+    const resp = await fetchTolgee(url);
+
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    return resp;
+};

libs/datasource/i18n/src/scripts/download.ts

@@ -0,0 +1,134 @@
+/* eslint-disable no-console */
+// cSpell:ignore Tolgee
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { format } from 'prettier';
+import { getAllProjectLanguages, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
+
+const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
+
+const countKeys = (obj: TranslationRes) => {
+    let count = 0;
+    Object.entries(obj).forEach(([key, value]) => {
+        if (typeof value === 'string') {
+            count++;
+        } else {
+            count += countKeys(value);
+        }
+    });
+    return count;
+};
+
+const getBaseTranslations = async (baseLanguage: { tag: string }) => {
+    try {
+        const baseTranslationsStr = await fs.readFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            { encoding: 'utf8' }
+        );
+        const baseTranslations = JSON.parse(baseTranslationsStr);
+        return baseTranslations;
+    } catch (e) {
+        console.error('base language:', JSON.stringify(baseLanguage));
+        console.error('Failed to read base language', e);
+        const translations = await getRemoteTranslations(baseLanguage.tag);
+        await fs.writeFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            JSON.stringify(translations, null, 4)
+        );
+    }
+};
+
+const main = async () => {
+    console.log('Loading project languages...');
+    const languages = await getAllProjectLanguages();
+    const baseLanguage = languages.find(language => language.base);
+    if (!baseLanguage) {
+        console.error(JSON.stringify(languages));
+        throw new Error('Could not find base language');
+    }
+    console.log(
+        `Loading ${baseLanguage.tag} languages translations as base...`
+    );
+
+    const baseTranslations = await getBaseTranslations(baseLanguage);
+    const baseKeyNum = countKeys(baseTranslations);
+    const languagesWithTranslations = await Promise.all(
+        languages.map(async language => {
+            console.log(`Loading ${language.tag} translations...`);
+            const translations = await getRemoteTranslations(language.tag);
+            const keyNum = countKeys(translations);
+
+            return {
+                ...language,
+                translations,
+                completeRate: keyNum / baseKeyNum,
+            };
+        })
+    );
+
+    const availableLanguages = languagesWithTranslations.filter(
+        language => language.completeRate > 0
+    );
+
+    availableLanguages
+        // skip base language
+        .filter(i => !i.base)
+        .forEach(async language => {
+            await fs.writeFile(
+                path.resolve(RES_DIR, `${language.tag}.json`),
+                JSON.stringify(
+                    {
+                        // eslint-disable-next-line @typescript-eslint/naming-convention
+                        '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
+                            '',
+                        ...language.translations,
+                    },
+                    null,
+                    4
+                )
+            );
+        });
+
+    console.log('Generating meta data...');
+    const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+        // Run \`pnpm run download-resources\` to regenerate.
+        // To overwrite this, please overwrite ${path.basename(__filename)}
+        ${availableLanguages
+            .map(
+                language =>
+                    `import ${language.tag.replaceAll('-', '_')} from './${
+                        language.tag
+                    }.json'`
+            )
+            .join('\n')}
+
+        export const LOCALES = [
+            ${availableLanguages
+                .map(({ translations, ...language }) =>
+                    JSON.stringify({
+                        ...language,
+                        res: '__RES_PLACEHOLDER',
+                    }).replace(
+                        '"__RES_PLACEHOLDER"',
+                        language.tag.replaceAll('-', '_')
+                    )
+                )
+                .join(',\n')}
+        ] as const;
+        `;
+
+    await fs.writeFile(
+        path.resolve(RES_DIR, 'index.ts'),
+        format(code, {
+            parser: 'typescript',
+            singleQuote: true,
+            trailingComma: 'es5',
+            tabWidth: 4,
+            arrowParens: 'avoid',
+        })
+    );
+    console.log('Done');
+};
+
+main();

libs/datasource/i18n/src/scripts/request.ts

@@ -46,6 +46,7 @@ const withTolgee = (
                     argArray[1].headers = headers;
                 }
             }
+            // console.log('fetch', argArray);
             return target.apply(thisArg, argArray);
         },
     });

libs/datasource/i18n/src/scripts/sync.ts

@@ -1,30 +1,15 @@
+/* eslint-disable no-console */
 // cSpell:ignore Tolgee
 import { readFile } from 'fs/promises';
 import path from 'path';
-import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+import { addTagByKey, createsNewKey, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
 
 const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
 const BASE_LANGUAGES = 'en' as const;
 
 const DEPRECATED_TAG_NAME = 'unused' as const;
 
-interface TranslationRes {
-    [x: string]: string | TranslationRes;
-}
-
-const getRemoteTranslations = async (languages: string) => {
-    const translations = await getLanguagesTranslations(languages);
-    if (!(languages in translations)) {
-        console.log(translations);
-        throw new Error(
-            'Failed to get base languages translation! base languages: ' +
-                languages
-        );
-    }
-    // The assert is safe because we checked above
-    return translations[languages]!;
-};
-
 /**
  *
  * @example

libs/datasource/i18n/src/scripts/utils.ts

@@ -0,0 +1,3 @@
+export interface TranslationRes {
+    [x: string]: string | TranslationRes;
+}