Merge pull request #404 from toeverything/develop

workflow improve
2024-12-23 14:42:03 +03:00 · 2022-09-11 17:55:45 +08:00 · 2022-09-11 17:55:45 +08:00 · 20e0ee50c1
commit 20e0ee50c1
parent 7225ed9243 fc1b3f345c
19 changed files with 367 additions and 74 deletions
--- a/.github/deployment/Dockerfile-affine
+++ b/.github/deployment/Dockerfile-affine
@ -1,9 +1,9 @@
 FROM node:16-alpine as builder
 WORKDIR /app
+RUN apk add git && npm i -g pnpm@7
 COPY . .
 # RUN apk add g++ make python3 git libpng-dev
-RUN apk add git
-RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local --skip-nx-cache
+RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local --skip-nx-cache

 FROM node:16-alpine as relocate
 WORKDIR /app
--- a/.github/deployment/Dockerfile-affine-keck
+++ b/.github/deployment/Dockerfile-affine-keck
@ -1,9 +1,9 @@
 FROM node:16-alpine as builder
 WORKDIR /app
+RUN apk add git && npm i -g pnpm@7
 COPY . .
 # RUN apk add g++ make python3 git libpng-dev
-RUN apk add git
-RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local-keck --skip-nx-cache
+RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i  --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local-keck --skip-nx-cache

 FROM node:16-alpine as relocate
 WORKDIR /app
--- a/.github/deployment/Dockerfile-keck
+++ b/.github/deployment/Dockerfile-keck
@ -1,8 +1,8 @@
 FROM node:16-alpine as builder
 WORKDIR /app
+RUN apk add g++ make python3 git libpng-dev && npm i -g pnpm@7
 COPY . .
-RUN apk add g++ make python3 git libpng-dev
-RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:keck
+RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i  --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:keck

 FROM node:16-alpine as node_modules
 WORKDIR /app
--- a/.github/deployment/Dockerfile-lisa
+++ b/.github/deployment/Dockerfile-lisa
@ -1,9 +1,9 @@
 FROM node:16-alpine as builder
 WORKDIR /app
+RUN apk add git && npm i -g pnpm@7
 COPY . .
 # RUN apk add g++ make python3 git libpng-dev
-RUN apk add git
-RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus"  && pnpm run build
+RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i  --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus"  && pnpm run build

 FROM node:16-alpine as relocate
 WORKDIR /app
--- a/.github/deployment/Dockerfile-venus
+++ b/.github/deployment/Dockerfile-venus
@ -1,8 +1,8 @@
 FROM node:16-alpine as builder
 WORKDIR /app
+RUN apk add g++ make python3 git libpng-dev && npm i -g pnpm@7
 COPY . .
-RUN apk add g++ make python3 git libpng-dev
-RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:venus
+RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i  --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:venus

 FROM node:16-alpine as relocate
 WORKDIR /app
--- a/.github/workflows/languages-download.yml
+++ b/.github/workflows/languages-download.yml
@ -0,0 +1,84 @@
+name: Download Languages Resources
+
+on:
+    schedule:
+      - cron: "0 0 * * 5" # At 00:00(UTC) on Friday.
+    workflow_dispatch:
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+# See https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # The concurrency group contains the workflow name and the branch name for
+    # pull requests or the commit hash for any other events.
+    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+    cancel-in-progress: true
+
+jobs:
+    main:
+        strategy:
+            matrix:
+                node-version: [18]
+                os: [ubuntu-latest]
+        runs-on: ${{ matrix.os }}
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+
+            - name: Use pnpm
+              uses: pnpm/action-setup@v2
+              with:
+                  version: 7
+
+            - name: Use Node.js ${{ matrix.node-version }}
+              # https://github.com/actions/setup-node
+              uses: actions/setup-node@v3
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: 'pnpm'
+
+            - name: Install node modules
+              run: pnpm install
+
+            - name: Sync Languages
+              if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master'
+              working-directory: ./libs/datasource/i18n
+              run: pnpm run download-resources
+              env:
+                TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
+
+            - name: Push Branch
+              id: push
+              run: |
+                git add libs/datasource/i18n
+                # Do not proceed if there are no file differences
+                COMMIT=$(git rev-parse --verify origin/$TARGET_BRANCH || echo HEAD)
+                FILES_CHANGED=$(git diff-index --name-only --cached $COMMIT | wc -l)
+                if [[ "$FILES_CHANGED" = "0" ]]; then
+                  echo "No file changes detected."
+                  echo "::set-output name=skipPR::true"
+                  exit 0
+                fi
+                git config user.name 'github-actions[bot]'
+                git config user.email 'github-actions[bot]@users.noreply.github.com'
+                git commit --message 'feat(i18n): new translations'
+                git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY"
+                git push --force origin HEAD:$TARGET_BRANCH
+              env:
+                GITHUB_TOKEN: ${{ secrets.github_token }}
+                TARGET_BRANCH: bot/new-translations
+
+            - name: Get current date
+              id: date
+              run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+            # see https://github.com/repo-sync/pull-request
+            - name: Create Pull Request
+              if: steps.push.outputs.skipPR != 'true'
+              uses: repo-sync/pull-request@v2
+              with:
+                source_branch: 'bot/new-translations' # If blank, default: triggered branch
+                destination_branch: "develop"
+                pr_title: Update i18n (${{ steps.date.outputs.date }}) # Title of pull request
+                pr_label: 'data,bot' # Comma-separated list (no spaces)
+                github_token: ${{ secrets.GITHUB_TOKEN }}
--- a/README.md
+++ b/README.md
@ -138,6 +138,11 @@ We would also like to give thanks to open-source projects that make affine possi
 -   [MUI](https://github.com/mui/material-ui) -- Our most used graphic UI component library.
 -   Other [dependencies](https://github.com/toeverything/AFFiNE/network/dependencies)

+We use the following open source projects to help us build a better development experience:
+
+-   [nx](https://github.com/nrwl/nx) -- Awesome monorepo manager & build system
+-   [tolgee](https://github.com/tolgee/tolgee-platform) -- Elegant i18n collaborative editing platform
+
 Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone.

 # Contributors
--- a/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx
+++ b/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx
@ -227,7 +227,7 @@ export const CommandMenu = ({ editor, hooks, style }: CommandMenuProps) => {
            const nextBlock = await block.nextSibling();
            setTimeout(() => {
                editor.selectionManager.activeNodeByNodeId(nextBlock.id);
-            });
+            }, 100);
            if (block.blockProvider.isEmpty()) {
                block.remove();
            }
--- a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx
+++ b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx
@ -53,10 +53,10 @@ export const SettingsList = () => {
                                >
                                    {LOCALES.map(option => (
                                        <Option
-                                            key={option.value}
-                                            value={option.value}
+                                            key={option.tag}
+                                            value={option.tag}
                                        >
-                                            {option.text}
+                                            {option.originalName}
                                        </Option>
                                    ))}
                                </Select>
--- a/libs/datasource/i18n/package.json
+++ b/libs/datasource/i18n/package.json
@ -3,7 +3,8 @@
    "version": "0.0.1",
    "scripts": {
        "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts",
-        "sync-languages:check": "pnpm run sync-languages --check"
+        "sync-languages:check": "pnpm run sync-languages --check",
+        "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts"
    },
    "dependencies": {
        "i18next": "^21.9.1",
--- a/libs/datasource/i18n/src/base.json
+++ b/libs/datasource/i18n/src/base.json
@ -1,24 +0,0 @@
-{
-    "Sync to Disk": "Sync to Disk",
-    "Share": "Share",
-    "WarningTips": {
-        "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
-        "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
-        "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
-    },
-    "Layout": "Layout",
-    "Comment": "Comment",
-    "Settings": "Settings",
-    "ComingSoon": "Layout Settings Coming Soon...",
-    "Duplicate Page": "Duplicate Page",
-    "Copy Page Link": "Copy Page Link",
-    "Language": "Language",
-    "Clear Workspace": "Clear Workspace",
-    "Export As Markdown": "Export As Markdown",
-    "Export As HTML": "Export As HTML",
-    "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
-    "Import Workspace": "Import Workspace",
-    "Export Workspace": "Export Workspace",
-    "Last edited by": "Last edited by {{name}}",
-    "Logout": "Logout"
-}
--- a/libs/datasource/i18n/src/index.ts
+++ b/libs/datasource/i18n/src/index.ts
@ -4,8 +4,8 @@ import {
    initReactI18next,
    useTranslation,
 } from 'react-i18next';
-import en_US from './resources/en.json';
-import zh_CN from './resources/zh.json';
+import { LOCALES } from './resources';
+import type en_US from './resources/en.json';

 // See https://react.i18next.com/latest/typescript
 declare module 'react-i18next' {
@ -21,22 +21,19 @@ declare module 'react-i18next' {

 const STORAGE_KEY = 'i18n_lng';

-const LOCALES = [
-    { value: 'en', text: 'English', res: en_US },
-    { value: 'zh', text: '简体中文', res: zh_CN },
-] as const;
+export { i18n, useTranslation, I18nProvider, LOCALES };

 const resources = LOCALES.reduce<Resource>(
-    (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }),
+    (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }),
    {}
 );

-const fallbackLng = LOCALES[0].value;
+const fallbackLng = LOCALES[0].tag;
 const standardizeLocale = (language: string) => {
-    if (LOCALES.find(locale => locale.value === language)) return language;
+    if (LOCALES.find(locale => locale.tag === language)) return language;
    if (
        LOCALES.find(
-            locale => locale.value === language.slice(0, 2).toLowerCase()
+            locale => locale.tag === language.slice(0, 2).toLowerCase()
        )
    )
        return language;
@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => {
 });

 const I18nProvider = I18nextProvider;
-
-export { i18n, useTranslation, I18nProvider, LOCALES };
--- a/libs/datasource/i18n/src/resources/index.ts
+++ b/libs/datasource/i18n/src/resources/index.ts
@ -0,0 +1,28 @@
+// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+// Run `pnpm run download-resources` to regenerate.
+// To overwrite this, please overwrite download.ts
+import en from './en.json';
+import zh_Hans from './zh-Hans.json';
+
+export const LOCALES = [
+    {
+        id: 1000016008,
+        name: 'English',
+        tag: 'en',
+        originalName: 'English',
+        flagEmoji: '🇬🇧',
+        base: true,
+        completeRate: 1,
+        res: en,
+    },
+    {
+        id: 1000016009,
+        name: 'Simplified Chinese',
+        tag: 'zh-Hans',
+        originalName: '简体中文',
+        flagEmoji: '🇨🇳',
+        base: false,
+        completeRate: 1,
+        res: zh_Hans,
+    },
+] as const;
--- a/libs/datasource/i18n/src/resources/zh-Hans.json
+++ b/libs/datasource/i18n/src/resources/zh-Hans.json
@ -1,4 +1,5 @@
 {
+    "// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.": "",
    "Sync to Disk": "同步到磁盘",
    "Share": "分享",
    "WarningTips": {
--- a/libs/datasource/i18n/src/scripts/api.ts
+++ b/libs/datasource/i18n/src/scripts/api.ts
@ -5,15 +5,65 @@ import { fetchTolgee } from './request';
 * Returns all project languages
 *
 * See https://tolgee.io/api#operation/getAll_6
+ * @example
+ * ```ts
+ * const languages = [
+ *   {
+ *     id: 1000016008,
+ *     name: 'English',
+ *     tag: 'en',
+ *     originalName: 'English',
+ *     flagEmoji: '🇬🇧',
+ *     base: true
+ *   },
+ *   {
+ *     id: 1000016013,
+ *     name: 'Spanish',
+ *     tag: 'es',
+ *     originalName: 'español',
+ *     flagEmoji: '🇪🇸',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016009,
+ *     name: 'Simplified Chinese',
+ *     tag: 'zh-Hans',
+ *     originalName: '简体中文',
+ *     flagEmoji: '🇨🇳',
+ *     base: false
+ *   },
+ *   {
+ *     id: 1000016012,
+ *     name: 'Traditional Chinese',
+ *     tag: 'zh-Hant',
+ *     originalName: '繁體中文',
+ *     flagEmoji: '🇭🇰',
+ *     base: false
+ *   }
+ * ]
+ * ```
 */
-export const getAllProjectLanguages = async () => {
-    const url = '/languages?size=1000';
+export const getAllProjectLanguages = async (size = 1000) => {
+    const url = `/languages?size=${size}`;
    const resp = await fetchTolgee(url);
    if (resp.status < 200 || resp.status >= 300) {
        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
    }
-    const json = await resp.json();
-    return json;
+    const json: {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _embedded: {
+            languages: {
+                id: number;
+                name: string;
+                tag: string;
+                originalName: string;
+                flagEmoji: string;
+                base: boolean;
+            }[];
+        };
+        page: unknown;
+    } = await resp.json();
+    return json._embedded.languages;
 };

 /**
@ -48,6 +98,16 @@ export const getLanguagesTranslations = async <T extends string>(
    return json;
 };

+export const getRemoteTranslations = async (languages: string) => {
+    const translations = await getLanguagesTranslations(languages);
+    if (!(languages in translations)) {
+        return {};
+    }
+    // The assert is safe because we checked above
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return translations[languages]!;
+};
+
 /**
 * Creates new key
 *
@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => {
    // const keyId =
    // addTag(keyId, tag);
 };
+
+/**
+ * Exports data
+ *
+ * See https://tolgee.io/api#operation/export_1
+ */
+export const exportResources = async () => {
+    const url = `/export`;
+    const resp = await fetchTolgee(url);
+
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    return resp;
+};
--- a/libs/datasource/i18n/src/scripts/download.ts
+++ b/libs/datasource/i18n/src/scripts/download.ts
@ -0,0 +1,134 @@
+/* eslint-disable no-console */
+// cSpell:ignore Tolgee
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { format } from 'prettier';
+import { getAllProjectLanguages, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
+
+const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
+
+const countKeys = (obj: TranslationRes) => {
+    let count = 0;
+    Object.entries(obj).forEach(([key, value]) => {
+        if (typeof value === 'string') {
+            count++;
+        } else {
+            count += countKeys(value);
+        }
+    });
+    return count;
+};
+
+const getBaseTranslations = async (baseLanguage: { tag: string }) => {
+    try {
+        const baseTranslationsStr = await fs.readFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            { encoding: 'utf8' }
+        );
+        const baseTranslations = JSON.parse(baseTranslationsStr);
+        return baseTranslations;
+    } catch (e) {
+        console.error('base language:', JSON.stringify(baseLanguage));
+        console.error('Failed to read base language', e);
+        const translations = await getRemoteTranslations(baseLanguage.tag);
+        await fs.writeFile(
+            path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+            JSON.stringify(translations, null, 4)
+        );
+    }
+};
+
+const main = async () => {
+    console.log('Loading project languages...');
+    const languages = await getAllProjectLanguages();
+    const baseLanguage = languages.find(language => language.base);
+    if (!baseLanguage) {
+        console.error(JSON.stringify(languages));
+        throw new Error('Could not find base language');
+    }
+    console.log(
+        `Loading ${baseLanguage.tag} languages translations as base...`
+    );
+
+    const baseTranslations = await getBaseTranslations(baseLanguage);
+    const baseKeyNum = countKeys(baseTranslations);
+    const languagesWithTranslations = await Promise.all(
+        languages.map(async language => {
+            console.log(`Loading ${language.tag} translations...`);
+            const translations = await getRemoteTranslations(language.tag);
+            const keyNum = countKeys(translations);
+
+            return {
+                ...language,
+                translations,
+                completeRate: keyNum / baseKeyNum,
+            };
+        })
+    );
+
+    const availableLanguages = languagesWithTranslations.filter(
+        language => language.completeRate > 0
+    );
+
+    availableLanguages
+        // skip base language
+        .filter(i => !i.base)
+        .forEach(async language => {
+            await fs.writeFile(
+                path.resolve(RES_DIR, `${language.tag}.json`),
+                JSON.stringify(
+                    {
+                        // eslint-disable-next-line @typescript-eslint/naming-convention
+                        '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
+                            '',
+                        ...language.translations,
+                    },
+                    null,
+                    4
+                ) + '\n'
+            );
+        });
+
+    console.log('Generating meta data...');
+    const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+        // Run \`pnpm run download-resources\` to regenerate.
+        // To overwrite this, please overwrite ${path.basename(__filename)}
+        ${availableLanguages
+            .map(
+                language =>
+                    `import ${language.tag.replaceAll('-', '_')} from './${
+                        language.tag
+                    }.json'`
+            )
+            .join('\n')}
+
+        export const LOCALES = [
+            ${availableLanguages
+                .map(({ translations, ...language }) =>
+                    JSON.stringify({
+                        ...language,
+                        res: '__RES_PLACEHOLDER',
+                    }).replace(
+                        '"__RES_PLACEHOLDER"',
+                        language.tag.replaceAll('-', '_')
+                    )
+                )
+                .join(',\n')}
+        ] as const;
+        `;
+
+    await fs.writeFile(
+        path.resolve(RES_DIR, 'index.ts'),
+        format(code, {
+            parser: 'typescript',
+            singleQuote: true,
+            trailingComma: 'es5',
+            tabWidth: 4,
+            arrowParens: 'avoid',
+        })
+    );
+    console.log('Done');
+};
+
+main();
--- a/libs/datasource/i18n/src/scripts/request.ts
+++ b/libs/datasource/i18n/src/scripts/request.ts
@ -46,6 +46,7 @@ const withTolgee = (
                    argArray[1].headers = headers;
                }
            }
+            // console.log('fetch', argArray);
            return target.apply(thisArg, argArray);
        },
    });
--- a/libs/datasource/i18n/src/scripts/sync.ts
+++ b/libs/datasource/i18n/src/scripts/sync.ts
@ -1,30 +1,20 @@
+/* eslint-disable no-console */
 // cSpell:ignore Tolgee
 import { readFile } from 'fs/promises';
 import path from 'path';
-import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+import { addTagByKey, createsNewKey, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';

-const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
+const BASE_JSON_PATH = path.resolve(
+    process.cwd(),
+    'src',
+    'resources',
+    'en.json'
+);
 const BASE_LANGUAGES = 'en' as const;

 const DEPRECATED_TAG_NAME = 'unused' as const;

-interface TranslationRes {
-    [x: string]: string | TranslationRes;
-}
-
-const getRemoteTranslations = async (languages: string) => {
-    const translations = await getLanguagesTranslations(languages);
-    if (!(languages in translations)) {
-        console.log(translations);
-        throw new Error(
-            'Failed to get base languages translation! base languages: ' +
-                languages
-        );
-    }
-    // The assert is safe because we checked above
-    return translations[languages]!;
-};
-
 /**
 *
 * @example
--- a/libs/datasource/i18n/src/scripts/utils.ts
+++ b/libs/datasource/i18n/src/scripts/utils.ts
@ -0,0 +1,3 @@
+export interface TranslationRes {
+    [x: string]: string | TranslationRes;
+}