mirego/accent
1
1
Fork 0
mirror of https://github.com/mirego/accent.git synced 2024-08-17 06:50:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix machine translation auto detect language

Browse Source
This commit is contained in:
Simon Prévost 2023-11-23 14:24:36 -05:00
parent 782594fb9b
commit ebaa67ab36
41 changed files with 246 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/accent.ex
View File

@ -10,6 +10,7 @@ defmodule Accent do
Accent.Repo,
{Oban, oban_config()},
Accent.Vault,
{Cachex, name: :language_tool_cache, limit: 10_000},
{LanguageTool.Server, language_tool_config()},
{TelemetryUI, Accent.TelemetryUI.config()},
{Phoenix.PubSub, [name: Accent.PubSub, adapter: Phoenix.PubSub.PG2]}

2
lib/accent/schemas/translation.ex
View File

@ -52,7 +52,7 @@ defmodule Accent.Translation do
cast(struct, params, @optional_fields)
end
@spec to_langue_entry(map(), map(), boolean, String.t()) :: Langue.Entry.t()
@spec to_langue_entry(map(), map() | nil, boolean, String.t()) :: Langue.Entry.t()
def to_langue_entry(translation, master_translation, is_master, language_slug) do
%Langue.Entry{
id: translation.id,

4
lib/graphql/resolvers/lint.ex
View File

@ -2,7 +2,6 @@ defmodule Accent.GraphQL.Resolvers.Lint do
@moduledoc false
import Absinthe.Resolution.Helpers, only: [batch: 3]
alias Accent.Language
alias Accent.Plugs.GraphQLContext
alias Accent.Repo
alias Accent.Scopes.Revision, as: RevisionScope
@ -11,8 +10,7 @@ defmodule Accent.GraphQL.Resolvers.Lint do
require Ecto.Query
@spec lint_translation(Translation.t(), map(), GraphQLContext.t()) ::
{:ok, Paginated.t(Language.Entry.t())}
@spec lint_translation(Translation.t(), map(), GraphQLContext.t()) :: {:middleware, Absinthe.Middleware.Batch, any()}
def lint_translation(translation, args, resolution) do
batch({__MODULE__, :preload_translations}, translation, fn batch_results ->
translation = Map.get(batch_results, translation.id)

10
lib/graphql/resolvers/machine_translation.ex
View File

@ -12,11 +12,11 @@ defmodule Accent.GraphQL.Resolvers.MachineTranslation do
@spec translate_text(
Project.t(),
%{text: String.t(), source_language_slug: String.t(), target_language_slug: String.t()},
%{text: String.t(), source_language_slug: String.t() | nil, target_language_slug: String.t()},
GraphQLContext.t()
) :: nil
) :: {:ok, %{error: nil | binary(), provider: atom(), text: binary()}}
def translate_text(project, args, _info) do
source_language = slug_language(project.id, args.source_language_slug)
source_language = args[:source_language_slug] && slug_language(project.id, args.source_language_slug)
target_language = slug_language(project.id, args.target_language_slug)
result = %{
@ -28,8 +28,8 @@ defmodule Accent.GraphQL.Resolvers.MachineTranslation do
result =
case MachineTranslations.translate(
[%{value: args.text}],
source_language,
target_language,
source_language && source_language.slug,
target_language.slug,
project.machine_translations_config
) do
[%{value: text}] -> %{result | text: text}

2
lib/graphql/types/project.ex
View File

@ -78,7 +78,7 @@ defmodule Accent.GraphQL.Types.Project do
field :translated_text, :project_translated_text do
arg(:text, non_null(:string))
arg(:source_language_slug, non_null(:string))
arg(:source_language_slug, :string)
arg(:target_language_slug, non_null(:string))
resolve(

2
lib/langue/formatter/simple_json/simple_json.ex
View File

@ -7,5 +7,5 @@ defmodule Langue.Formatter.SimpleJson do
parser: Langue.Formatter.SimpleJson.Parser,
serializer: Langue.Formatter.SimpleJson.Serializer
def placeholder_regex, do: ~r/{{?[^}]*}?}/
def placeholder_regex, do: Langue.Formatter.Json.placeholder_regex()
end

22
lib/lint/checks/spelling.ex
View File

@ -13,21 +13,33 @@ defmodule Accent.Lint.Checks.Spelling do
LanguageTool.ready?() and
is_binary(entry.value) and
not String.match?(entry.value, ~r/MMM|YYY|HH|AA/i) and
not String.starts_with?(entry.value, "{") and
((!entry.is_master and entry.value !== entry.master_value) or entry.is_master) and
String.length(entry.value) < 100 and String.length(entry.value) > 3
end
@impl true
def check(entry) do
{matches, markups} =
matches =
case LanguageTool.check(entry.language_slug, entry.value, placeholder_regex: entry.placeholder_regex) do
%{"matches" => matches, "markups" => markups} -> {matches, markups}
_ -> {[], []}
%{"matches" => matches} -> matches
_ -> []
end
matches =
case matches do
[%{"offset" => 0} | rest] ->
if String.starts_with?(entry.value, "{") do
rest
else
matches
end
matches ->
matches
end
for match <- matches do
offset = match["offset"] + length(markups)
offset = match["offset"]
replacement =
case match["replacements"] do

2
lib/lint/lint.ex
View File

@ -40,7 +40,7 @@ defmodule Accent.Lint do
fn {_, check} -> check.enabled?() end
)
@spec lint(list(entry), Config.t()) :: list(map())
@spec lint(list(entry), Config.t()) :: list({entry, list(map())})
def lint(entries, config \\ %Config{}) do
Enum.map(entries, &entry_to_messages(&1, config))
end

18
lib/machine_translations/machine_translations.ex
View File

@ -23,7 +23,7 @@ defmodule Accent.MachineTranslations do
end
@spec translate([map()], String.t(), String.t(), struct()) :: [map()] | {:error, any()}
def translate(entries, source_language, target_language, config) do
def translate(entries, source_language_slug, target_language_slug, config) do
provider = provider_from_config(config)
entries
@ -32,7 +32,7 @@ defmodule Accent.MachineTranslations do
|> Enum.reduce_while([], fn chunked_entries, acc ->
values = Enum.map(chunked_entries, & &1.value)
case Provider.translate(provider, values, source_language.slug, target_language.slug) do
case Provider.translate(provider, values, source_language_slug, target_language_slug) do
{:ok, translated_values} ->
translated_entries =
chunked_entries
@ -53,8 +53,10 @@ defmodule Accent.MachineTranslations do
end)
end
@spec map_source_and_target(String.t() | nil, String.t(), list(String.t())) ::
{:ok, {String.t(), String.t()}} | {:error, atom()}
def map_source_and_target(source, target, supported_languages) do
source = String.downcase(source)
source = source && String.downcase(source)
target = String.downcase(target)
source =
@ -64,13 +66,15 @@ defmodule Accent.MachineTranslations do
if target in supported_languages, do: target, else: fallback_split_lanugage_slug(target, supported_languages)
cond do
is_nil(source) and is_nil(target) -> {:error, :unsupported_source_and_target}
is_nil(source) -> {:error, :unsupported_source}
is_nil(target) -> {:error, :unsupported_target}
source === :unsupported and target === :unsupported -> {:error, :unsupported_source_and_target}
source === :unsupported -> {:error, :unsupported_source}
target === :unsupported -> {:error, :unsupported_target}
true -> {:ok, {source, target}}
end
end
defp fallback_split_lanugage_slug(nil, _supported_languages), do: nil
defp fallback_split_lanugage_slug(language, supported_languages) do
prefix =
case String.split(language, "-", parts: 2) do
@ -78,7 +82,7 @@ defmodule Accent.MachineTranslations do
_ -> nil
end
if prefix in supported_languages, do: prefix
if prefix in supported_languages, do: prefix, else: :unsupported
end
defp provider_from_config(nil), do: %Provider.NotImplemented{}

4
lib/movement/machine_translations.ex
View File

@ -19,8 +19,8 @@ defmodule Movement.MachineTranslations do
|> Enum.filter(& &1.machine_translations_enabled)
|> Enum.map(&Operation.to_langue_entry(&1, master_revision.id === &1.revision_id, language_slug(revision)))
|> MachineTranslations.translate(
%{slug: language_slug(master_revision)},
%{slug: language_slug(revision)},
language_slug(master_revision),
language_slug(revision),
project.machine_translations_config
)
|> case do

8
lib/web/controllers/machine_translations_controller.ex
View File

@ -59,8 +59,8 @@ defmodule Accent.MachineTranslationsController do
case Accent.MachineTranslations.translate(
entries,
conn.assigns[:source_language],
conn.assigns[:target_language],
conn.assigns[:source_language].slug,
conn.assigns[:target_language].slug,
conn.assigns[:project].machine_translations_config
) do
entries when is_list(entries) ->
@ -109,8 +109,8 @@ defmodule Accent.MachineTranslationsController do
entries =
Accent.MachineTranslations.translate(
conn.assigns[:movement_context].entries,
conn.assigns[:source_language],
conn.assigns[:target_language],
conn.assigns[:source_language].slug,
conn.assigns[:target_language].slug,
conn.assigns[:project].machine_translations_config
)

3
mix.exs
View File

@ -59,6 +59,9 @@ defmodule Accent.Mixfile do
# Spelling interop with Java runtime
{:exile, "~> 0.7"},
# Cache
{:cachex, "~> 3.6"},
# Phoenix data helpers
{:phoenix_ecto, "~> 4.0"},
{:scrivener_ecto, "~> 2.0"},

5
mix.lock
View File

@ -7,6 +7,7 @@
"bamboo_smtp": {:hex, :bamboo_smtp, "4.2.2", "e9f57a2300df9cb496c48751bd7668a86a2b89aa2e79ccaa34e0c46a5f64c3ae", [:mix], [{:bamboo, "~> 2.2.0", [hex: :bamboo, repo: "hexpm", optional: false]}, {:gen_smtp, "~> 1.2.0", [hex: :gen_smtp, repo: "hexpm", optional: false]}], "hexpm", "28cac2ec8adaae02aed663bf68163992891a3b44cfd7ada0bebe3e09bed7207f"},
"bandit": {:hex, :bandit, "1.1.0", "1414e65916229d4ee0914f6d4e7f8ec16c6f2d90e01ad5174d89e90baa577625", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4891fb2f48a83445da70a4e949f649a9b4032310f1f640f4a8a372bc91cece18"},
"bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
"cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"},
"canada": {:hex, :canada, "2.0.0", "ce5e058f576a0625959fc5427fcde15311fb28a5ebc13775eafd13468ad16553", [:mix], [], "hexpm", "49a648c48d8b0864380f38f02a7f316bd30fd45602205c48197432b5225d8596"},
"canary": {:hex, :canary, "1.1.1", "4138d5e05db8497c477e4af73902eb9ae06e49dceaa13c2dd9f0b55525ded48b", [:mix], [{:canada, "~> 1.0.1", [hex: :canada, repo: "hexpm", optional: false]}, {:ecto, ">= 1.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f348d9848693c830a65b707bba9e4dfdd6434e8c356a8d4477e4535afb0d653b"},
"castore": {:hex, :castore, "1.0.4", "ff4d0fb2e6411c0479b1d965a814ea6d00e51eb2f58697446e9c41a97d940b28", [:mix], [], "hexpm", "9418c1b8144e11656f0be99943db4caf04612e3eaecefb5dae9a2a87565584f8"},
@ -29,6 +30,7 @@
"elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"},
"erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
"erlsom": {:hex, :erlsom, "1.5.1", "c8fe2babd33ff0846403f6522328b8ab676f896b793634cfe7ef181c05316c03", [:rebar3], [], "hexpm", "7965485494c5844dd127656ac40f141aadfa174839ec1be1074e7edf5b4239eb"},
"eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
"excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"},
"exile": {:hex, :exile, "0.7.0", "a07228b191c7233f48e225289cc512dd268b54b5e799d952d03b6ee6db3b7ba5", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f61ecfc4e0b2e9cc1ef177adf6b00a9bed2e8c2f4a2c2cfff0f3a28ae965720e"},
"fast_yaml": {:git, "https://github.com/processone/fast_yaml.git", "e789f68895f71b7ad31057177810ca0161bf790e", [ref: "e789f68895f71b7ad31057177810ca0161bf790e"]},
@ -44,6 +46,7 @@
"jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
"jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"},
"jsone": {:hex, :jsone, "1.8.1", "6bc74d3863d55d420077346da97c601711017a057f2fd1df65d6d65dd562fbab", [:rebar3], [], "hexpm", "c78918124148c51a7a84c678e39bbc6281f8cb582f1d88584628a98468e99738"},
"jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"},
"meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"},
@ -80,6 +83,7 @@
"scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"},
"scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"},
"sentry": {:hex, :sentry, "7.2.5", "570db92c3bbacd6ad02ac81cba8ac5af11235a55d65ac4375e3ec833975b83d3", [:mix], [{:hackney, "1.6.5 or ~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "ea84ed6848505ff2a246567df562f465d2b34c317d3ecba7c7df58daa56e5e5d"},
"sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
"styler": {:hex, :styler, "0.10.1", "f39d632c6474023c5e65434db3c0007405d1262cf45f9f886ce251372aaea294", [:mix], [], "hexpm", "e341d7d11b749d05e277b7971dad931c7eb6718f8b14a54fde728e8296791870"},
"table": {:hex, :table, "0.1.2", "87ad1125f5b70c5dea0307aa633194083eb5182ec537efc94e96af08937e14a8", [:mix], [], "hexpm", "7e99bc7efef806315c7e65640724bf165c3061cdc5d854060f74468367065029"},
@ -99,6 +103,7 @@
"ueberauth_google": {:hex, :ueberauth_google, "0.12.1", "90cf49743588193334f7a00da252f92d90bfd178d766c0e4291361681fafec7d", [:mix], [{:oauth2, "~> 1.0 or ~> 2.0", [hex: :oauth2, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.10.0", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm", "7f7deacd679b2b66e3bffb68ecc77aa1b5396a0cbac2941815f253128e458c38"},
"ueberauth_microsoft": {:hex, :ueberauth_microsoft, "0.23.0", "5c78e02a83d821ee45f96216bb6140ba688cc79b8b26e7ff438e3abe24615e1d", [:mix], [{:oauth2, "~> 1.0 or ~> 2.0", [hex: :oauth2, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.7", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm", "0c08d98203e6d3069f30306f09a6cb55b95c2bda94d6f8e90f05bd442ee96b82"},
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
"unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"},
"vega_lite": {:hex, :vega_lite, "0.1.8", "7f6119126ecaf4bc2c1854084370d7091424f5cce4795fbac044eee9963f0752", [:mix], [{:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: false]}], "hexpm", "6c8a9271f850612dd8a90de8d1ebd433590ed07ffef76fc2397c240dc04d3fdc"},
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
"websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"},

12
test/machine_translations/machine_translations_test.exs
View File

@ -21,8 +21,8 @@ defmodule AccentTest.MachineTranslations do
end)
entries = [%Langue.Entry{value: "Test", value_type: "string", key: "."}]
source_language = %{slug: "fr"}
target_language = %{slug: "en"}
source_language = "fr"
target_language = "en"
provider_config = %{"key" => Jason.encode!(%{"project_id" => "1234", "type" => "service_account"})}
config = %{"provider" => "google_translate", "config" => provider_config}
@ -49,8 +49,8 @@ defmodule AccentTest.MachineTranslations do
end)
entries = [%Langue.Entry{value: "Test %{placeholder} bla", value_type: "string", key: "."}]
source_language = %{slug: "fr"}
target_language = %{slug: "en"}
source_language = "fr"
target_language = "en"
provider_config = %{"key" => Jason.encode!(%{"project_id" => "1234", "type" => "service_account"})}
config = %{"provider" => "google_translate", "config" => provider_config}
@ -65,8 +65,8 @@ defmodule AccentTest.MachineTranslations do
end)
entries = [%Langue.Entry{value: "Test", value_type: "string", key: "."}]
source_language = %{slug: "fr"}
target_language = %{slug: "en"}
source_language = "fr"
target_language = "en"
provider_config = %{"key" => Jason.encode!(%{"project_id" => "1234", "type" => "service_account"})}
config = %{"provider" => "google_translate", "config" => provider_config}

38
vendor/language_tool/lib/language_tool.ex vendored
View File

@ -3,21 +3,31 @@ defmodule LanguageTool do
def check(lang, text, opts \\ []) do
if lang in list_languages() do
metadata = %{language_code: lang, text_length: String.length(text)}
cache_key = cache_key([lang, text])
:telemetry.span(
[:accent, :language_tool, :check],
metadata,
fn ->
placeholder_regex = Keyword.get(opts, :placeholder_regex)
annotated_text = LanguageTool.AnnotatedText.build(text, placeholder_regex)
case Cachex.get!(:language_tool_cache, cache_key) do
nil ->
metadata = %{language_code: lang, cache_key: cache_key, text_length: String.length(text)}
result =
GenServer.call(LanguageTool.Server, {:check, lang, Jason.encode!(%{items: annotated_text})}, :infinity)
:telemetry.span(
[:accent, :language_tool, :check],
metadata,
fn ->
placeholder_regex = Keyword.get(opts, :placeholder_regex)
annotated_text = LanguageTool.AnnotatedText.build(text, placeholder_regex)
{result, metadata}
end
)
result =
GenServer.call(LanguageTool.Server, {:check, lang, Jason.encode!(%{items: annotated_text})}, :infinity)
Cachex.set!(:language_tool_cache, cache_key, result)
{result, metadata}
end
)
result ->
result
end
else
empty_matches(lang, text, :unsupported_language)
end
@ -25,6 +35,10 @@ defmodule LanguageTool do
_ -> empty_matches(lang, text, :check_internal_error)
end
defp cache_key(contents) do
:erlang.md5(contents)
end
defp empty_matches(lang, text, error) do
%{"error" => error, "language" => lang, "matches" => [], "text" => text, "markups" => []}
end

46
vendor/language_tool/lib/language_tool/annotated_text.ex vendored
View File

@ -1,16 +1,40 @@
defmodule LanguageTool.AnnotatedText do
@moduledoc false
def build(input, regex) do
matches = if regex, do: Regex.scan(regex, input, return: :index), else: []
matches = scan_entry_regex(input, regex)
# Ignore HTML
matches = matches ++ Regex.scan(~r/<[^>]*>/, input, return: :index)
matches = matches ++ scan_html(input)
# Ignore % and $ often used as placeholders
matches = matches ++ Regex.scan(~r/[%$][\w\d]+/, input, return: :index)
matches = Enum.sort_by(matches, fn [{match_index, _}] -> match_index end)
matches = matches ++ scan_placeholders(input)
matches = Enum.sort_by(matches, fn {match_index, _, _} -> match_index end)
split_tokens(input, matches, 0, [])
end
defp scan_entry_regex(_input, nil), do: []
defp scan_entry_regex(input, regex) do
regex
|> Regex.scan(input, return: :index)
|> List.flatten()
|> Enum.map(fn {index, length} -> {index, length, "x"} end)
end
defp scan_html(input) do
~r/<[^>]*>/
|> Regex.scan(input, return: :index)
|> List.flatten()
|> Enum.map(fn {index, length} -> {index, length, ""} end)
end
defp scan_placeholders(input) do
~r/[%$][a-zA-Z0-9]+/
|> Regex.scan(input, return: :index)
|> List.flatten()
|> Enum.map(fn {index, length} -> {index, length, "x"} end)
end
defp split_tokens(input, [], position, acc) do
to_add =
case binary_slice(input, position..byte_size(input)) do
@ -24,18 +48,24 @@ defmodule LanguageTool.AnnotatedText do
acc ++ to_add
end
defp split_tokens(input, [[{start_index, match_length}] | matches], position, acc) do
defp split_tokens(input, [{start_index, match_length, markup_as} | matches], position, acc) do
text_before =
if position !== start_index do
if position !== start_index and position < start_index do
case binary_slice(input, position..(start_index - 1)) do
"" -> []
text_before -> [%{text: String.trim_leading(text_before)}]
text_before -> [%{text: text_before}]
end
else
[]
end
markup = binary_slice(input, start_index, match_length)
split_tokens(input, matches, start_index + match_length, acc ++ text_before ++ [%{markup: markup}])
split_tokens(
input,
matches,
start_index + match_length,
acc ++ text_before ++ [%{markup: markup, markupAs: markup_as}]
)
end
end

7
vendor/language_tool/priv/native/languagetool/app/src/main/kotlin/com/mirego/accent/languagetool/App.kt vendored
View File

@ -18,7 +18,7 @@ import kotlinx.serialization.json.*
data class Base(val items: Array<Item>)
@Serializable
data class Item(val markup: String = "", val text: String = "")
data class Item(val markup: String = "", val text: String = "", val markupAs: String = "x")
fun main(args: Array<String>) {
val reader = BufferedReader(InputStreamReader(System.`in`))
@ -54,7 +54,6 @@ fun main(args: Array<String>) {
}
for (code in languages) {
val cache = ResultCache(10000, 600, TimeUnit.SECONDS)
val globalConfig = GlobalConfig()
val userConfig = UserConfig()
@ -63,7 +62,7 @@ fun main(args: Array<String>) {
Languages.getLanguageForShortCode(code),
ArrayList(),
null,
cache,
null,
globalConfig,
userConfig
)
@ -103,7 +102,7 @@ fun main(args: Array<String>) {
for (item in parsedText.items) {
if (item.markup != "") {
markups.add(item.markup)
annotatedBuilder.addMarkup(item.markup, "x");
annotatedBuilder.addMarkup(item.markup, item.markupAs);
} else {
annotatedBuilder.addText(item.text);
}

16
webapp/.editorconfig
View File

@ -1,16 +0,0 @@
# EditorConfig helps developers define and maintain consistent
# coding styles between different editors and IDEs
# editorconfig.org
root = true
[*]
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
indent_size = 2
[*.{diff,md}]
trim_trailing_whitespace = false

42
webapp/app/pods/components/commit-file/component.ts
View File

@ -115,12 +115,20 @@ export default class CommitFile extends Component<Args> {
@tracked
correctOnMerge = false;
@tracked
revision =
this.args.commitAction === 'merge' && this.args.revisions[1]
? this.args.revisions[1]
: this.args.revisions.find((revision: any) => revision.isMaster);
@tracked
revisionValue =
this.mappedRevisions.find(({value}) => value === this.revision) ||
(this.args.commitAction === 'merge' && this.mappedRevisions[1])
? this.mappedRevisions[1]
: this.mappedRevisions[0];
this.mappedRevisions.find(
({value}) => value === (this.revision && this.revision.id)
) || this.mappedRevisions[0];
@tracked
version: {id: string; tag: string} | null = null;
@tracked
versionValue =
@ -128,15 +136,14 @@ export default class CommitFile extends Component<Args> {
({value}) => value === (this.version && this.version.id)
) || this.mappedVersions[0];
@tracked
revision =
this.args.revisions.find((revision: any) => revision.isMaster) ||
(this.args.commitAction === 'merge' && this.args.revisions[1])
? this.args.revisions[1]
: this.args.revisions[0];
@tracked
version: {id: string; tag: string} | null = null;
get mappedRevisions(): Array<{label: string; value: string}> {
return this.args.revisions.map(
({id, language}: {id: string; language: {name: string}}) => ({
label: language.name,
value: id,
})
);
}
get hasVersions() {
return this.args.versions.length > 0;
@ -156,15 +163,6 @@ export default class CommitFile extends Component<Args> {
}));
}
get mappedRevisions(): Array<{label: string; value: string}> {
return this.args.revisions.map(
({id, language}: {id: string; language: {name: string}}) => ({
label: language.name,
value: id,
})
);
}
get mappedVersions(): Array<{label: string; value: string}> {
return [
{

2
webapp/app/pods/components/conflicts-list/item/component.ts
View File

@ -40,7 +40,7 @@ interface Args {
onCorrect: (conflict: any, textInput: string) => Promise<MutationResponse>;
onCopyTranslation: (
text: string,
sourceLanguageSlug: string,
sourceLanguageSlug: string | null,
targetLanguageSlug: string
) => Promise<{text: string | null}>;
}

4
webapp/app/pods/components/conflicts-list/item/related-translation/component.ts
View File

@ -20,8 +20,8 @@ export default class ConflictItemRelatedTranslation extends Component<Args> {
get revisionName() {
return (
this.args.translation.revision.name ||
this.args.translation.revision.language.name
this.args.translation.revision.slug ||
this.args.translation.revision.language.slug
);
}

29
webapp/app/pods/components/conflicts-list/item/related-translation/styles.scss
View File

@ -1,5 +1,6 @@
.item {
display: flex;
align-items: start;
flex-direction: row;
padding: 5px 0 3px;
color: var(--color-grey);
@ -7,23 +8,30 @@
}
.item-link {
display: inline-flex;
place-items: center;
flex-shrink: 0;
margin-right: 4px;
margin-right: 2px;
color: var(--color-black);
opacity: 0.6;
opacity: 0.8;
text-decoration: none;
font-weight: bold;
font-size: 12px;
padding: 1px 5px;
background: var(--background-light-highlight);
border-radius: var(--border-radius);
font-size: 11px;
transition: 0.2s ease-in-out;
transition-property: opacity;
&:focus,
&:hover {
opacity: 1;
color: var(--color-primary);
}
}
.item-text {
width: 100%;
display: flex;
align-items: start;
}
.item-text-content {
@ -38,19 +46,20 @@
.item-actions {
margin-left: 6px;
flex-shrink: 0;
flex-grow: 1;
flex-grow: 0;
}
.item-actions-flag,
.item-actions-link {
display: inline-flex;
align-items: center;
justify-content: center;
margin-left: 3px;
border-radius: 50%;
height: 17px;
width: 17px;
padding: 3px;
background: transparent;
color: var(--color-black);
background: var(--color-primary-opacity-20);
line-height: 1;
}
@ -93,6 +102,6 @@
}
.item-actions-icon {
width: 12px;
height: 12px;
width: 10px;
height: 10px;
}

37
webapp/app/pods/components/conflicts-list/item/related-translation/template.hbs
View File

@ -5,36 +5,13 @@
<div local-class='item-text'>
<span local-class='item-text-content'>{{this.text}}</span>
</div>
<div local-class='item-actions'>
<LinkTo
title={{t 'components.conflict_item.translation_tooltip'}}
class='tooltip tooltip--top'
@route='logged-in.project.translation'
@models={{array @project.id @translation.id}}
local-class='item-actions-link'
>
{{inline-svg 'assets/eye.svg' local-class='item-actions-icon'}}
</LinkTo>
{{#if (get @permissions 'machine_translations_translate')}}
<button title={{t 'components.conflict_item.translate_tooltip'}} class='tooltip tooltip--top' local-class='item-actions-link' {{on 'click' this.translate}}>
{{inline-svg 'assets/language.svg' local-class='item-actions-icon'}}
</button>
{{/if}}
{{#if @translation.isConflicted}}
<LinkTo
title={{t 'components.conflict_item.in_review_tooltip'}}
class='tooltip tooltip--top'
@route='logged-in.project.revision.conflicts'
@models={{array @project.id @translation.revision.id}}
@query={{hash query=@translation.id}}
local-class='item-actions-flag item-actions-flag--link'
>
{{t 'components.conflict_item.conflicted_label'}}
</LinkTo>
{{/if}}
<div local-class='item-actions'>
{{#if (get @permissions 'machine_translations_translate')}}
<button title={{t 'components.conflict_item.translate_tooltip'}} class='tooltip tooltip--top' local-class='item-actions-link' {{on 'click' this.translate}}>
{{inline-svg 'assets/language.svg' local-class='item-actions-icon'}}
</button>
{{/if}}
</div>
</div>
</div>

1
webapp/app/pods/components/conflicts-list/item/template.hbs
View File

@ -78,7 +78,6 @@
@prompts={{@prompts}}
@rtl={{this.revisionTextDirRtl}}
@text={{this.textInput}}
@sourceLanguageSlug={{this.revisionSlug}}
@onUpdatingText={{fn this.onUpdatingText}}
@onUpdateText={{fn this.onUpdateText}}
/>

6
webapp/app/pods/components/documents-list/item/component.ts
View File

@ -36,6 +36,12 @@ export default class DocumentsListItem extends Component<Args> {
@tracked
isUpdating = false;
get multipleRevisions() {
return (
this.args.project.revisions && this.args.project.revisions.length > 1
);
}
get lowPercentage() {
return this.correctedKeysPercentage < LOW_PERCENTAGE;
}

20
webapp/app/pods/components/documents-list/item/template.hbs
View File

@ -91,15 +91,17 @@
{{inline-svg '/assets/sync.svg' class='button-icon'}}
</LinkTo>
{{/if}}
{{#if (get @permissions 'merge')}}
<LinkTo
@route='logged-in.project.files.add-translations'
@models={{array @project.id @document.id}}
title={{t 'components.documents_list.merge'}}
class='tooltip tooltip--top button button--filled button--white button--iconOnly'
>
{{inline-svg '/assets/merge.svg' class='button-icon'}}
</LinkTo>
{{#if this.multipleRevisions}}
{{#if (get @permissions 'merge')}}
<LinkTo
@route='logged-in.project.files.add-translations'
@models={{array @project.id @document.id}}
title={{t 'components.documents_list.merge'}}
class='tooltip tooltip--top button button--filled button--white button--iconOnly'
>
{{inline-svg '/assets/merge.svg' class='button-icon'}}
</LinkTo>
{{/if}}
{{/if}}
{{#if (get @permissions 'machine_translations_translate')}}
<LinkTo

2
webapp/app/pods/components/inline-machine-translate/component.ts
View File

@ -8,7 +8,6 @@ import projectTranslateTextQuery from 'accent-webapp/queries/translate-text-proj
interface Args {
text: string;
sourceLanguageSlug: string;
project: {id: string};
onUpdatingText: () => void;
onUpdateText: (value: string) => void;
@ -29,7 +28,6 @@ export default class ImprovePrompt extends Component<Args> {
const variables = {
projectId: this.args.project.id,
text: this.args.text,
sourceLanguageSlug: this.args.sourceLanguageSlug,
targetLanguageSlug,
};

4
webapp/app/pods/components/inline-machine-translate/styles.scss
View File

@ -49,7 +49,7 @@
pointer-events: none;
right: 0;
display: flex;
gap: 4px;
gap: 0;
justify-content: flex-end;
align-items: center;
position: absolute;
@ -59,11 +59,11 @@
> button {
font-size: 11px;
flex-shrink: 0;
font-weight: normal;
}
> button:focus,
> button:hover {
background: var(--background-light);
transform: translate3d(0, 0, 0);
}

6
webapp/app/pods/components/inline-machine-translate/template.hbs
View File

@ -2,7 +2,11 @@
<div local-class='prompt-button' tabindex='0'>
<div local-class='prompt-button-quick-access' data-rtl={{@rtl}}>
{{#each @languages as |language|}}
<AsyncButton title={{language.name}} @onClick={{(perform this.submitTask language.slug)}} class='button button--iconOnly button--filled button--borderless button--white'>
<AsyncButton
title={{language.name}}
@onClick={{(perform this.submitTask language.slug)}}
class='tooltip tooltip--top button button--iconOnly button--borderless button--white'
>
{{language.slug}}
</AsyncButton>
{{/each}}

26
webapp/app/pods/components/lint-translations-page/item/component.ts
View File

@ -6,6 +6,12 @@ interface Args {
project: any;
}
const escape = document.createElement('textarea');
const escapeHTML = (html: string) => {
escape.textContent = html;
return escape.innerHTML;
};
export default class LintTranslationsPageItem extends Component<Args> {
translationKey = parsedKeyProperty(this.args.lintTranslation.translation.key);
@ -24,7 +30,7 @@ export default class LintTranslationsPageItem extends Component<Args> {
get annotatedText() {
let offsetTotal = 0;
return this.args.lintTranslation.messages
let text = this.args.lintTranslation.messages
.sort((a: any, b: any) => a.offset || 0 >= b.offset || 0)
.reduce((text: string, message: any) => {
if (message.length) {
@ -34,23 +40,24 @@ export default class LintTranslationsPageItem extends Component<Args> {
);
if (message.replacement) {
const replacement = `<span data-underline>${error}</span><strong>${message.replacement.label}</strong>`;
const replacement = `(span data-underline)${error}(/span)(strong)${message.replacement.label}(/strong)`;
offsetTotal += replacement.length - error.length;
return String(text).replace(error, replacement);
} else {
const replacement = `<span data-underline>${error}</span>`;
const replacement = `(span data-underline)${error}(/span)`;
offsetTotal += replacement.length - error.length;
return String(text).replace(error, replacement);
}
} else if (message.check === 'LEADING_SPACES') {
const replacement = `<span data-rect> </span>`;
const replacement = `(span data-rect) (/span)`;
offsetTotal += replacement.length - 1;
return String(text).replace(/^ /, replacement);
} else if (message.check === 'TRAILING_SPACE') {
const replacement = `<span data-rect> </span>`;
const replacement = `(span data-rect) (/span)`;
offsetTotal += replacement.length - 1;
return String(text).replace(/ $/, replacement);
@ -58,5 +65,14 @@ export default class LintTranslationsPageItem extends Component<Args> {
return text;
}
}, this.args.lintTranslation.messages[0].text);
text = escapeHTML(text);
text = text.replaceAll('(span data-underline)', '<span data-underline>');
text = text.replaceAll('(span data-rect)', '<span data-rect>');
text = text.replaceAll('(/span)', '</span>');
text = text.replaceAll('(/strong)', '</strong>');
text = text.replaceAll('(strong)', '<strong>');
return text;
}
}

1
webapp/app/pods/components/lint-translations-page/item/styles.scss
View File

@ -1,7 +1,6 @@
.wrapper {
display: flex;
flex-direction: column;
margin-bottom: 20px;
}
.details {

4
webapp/app/pods/components/lint-translations-page/styles.scss
View File

@ -21,6 +21,10 @@
);
}
.item {
margin-bottom: 20px;
}
.icon-warning {
margin-right: 8px;
stroke: var(--color-warning);

4
webapp/app/pods/components/lint-translations-page/template.hbs
View File

@ -15,7 +15,9 @@
</div>
{{#each @lintTranslations key='id' as |lintTranslation|}}
<LintTranslationsPage::Item @project={{@project}} @lintTranslation={{lintTranslation}} />
<div local-class='item'>
<LintTranslationsPage::Item @project={{@project}} @lintTranslation={{lintTranslation}} />
</div>
{{/each}}
</div>
{{else}}

1
webapp/app/pods/components/related-translations-list/item/template.hbs
View File

@ -62,7 +62,6 @@
@prompts={{@prompts}}
@rtl={{this.revisionTextDirRtl}}
@text={{this.editText}}
@sourceLanguageSlug={{this.revisionSlug}}
@onUpdatingText={{fn this.onUpdatingText}}
@onUpdateText={{fn this.onUpdateText}}
/>

1
webapp/app/pods/components/translation-edit/helpers/component.ts
View File

@ -2,7 +2,6 @@ import Component from '@glimmer/component';
interface Args {
revisions: any[];
sourceLanguageSlug: string;
}
export default class TranslationEditHelpers extends Component<Args> {

1
webapp/app/pods/components/translation-edit/helpers/template.hbs
View File

@ -5,7 +5,6 @@
@project={{@project}}
@text={{@text}}
@languages={{this.machineTranslationLanguages}}
@sourceLanguageSlug={{@sourceLanguageSlug}}
@onUpdatingText={{@onUpdatingText}}
@onUpdateText={{@onUpdateText}}
/>

1
webapp/app/pods/components/translation-edit/template.hbs
View File

@ -25,7 +25,6 @@
@prompts={{@prompts}}
@rtl={{this.revisionTextDirRtl}}
@text={{this.text}}
@sourceLanguageSlug={{this.revisionSlug}}
@onUpdatingText={{fn this.onUpdatingText}}
@onUpdateText={{fn this.onUpdateText}}
/>

2
webapp/app/pods/components/translations-list/item/styles.scss
View File

@ -2,7 +2,7 @@
transition: 0.2s ease-in-out;
transition-property: background, transform;
display: block;
margin: 0;
margin: 0 0 5px;
padding: 4px 10px;
border: 1px solid transparent;
border-radius: var(--border-radius);

1
webapp/app/pods/components/translations-list/item/template.hbs
View File

@ -101,7 +101,6 @@
@prompts={{@prompts}}
@rtl={{this.revisionTextDirRtl}}
@text={{this.editText}}
@sourceLanguageSlug={{this.revisionSlug}}
@onUpdatingText={{fn this.onUpdatingText}}
@onUpdateText={{fn this.onUpdateText}}
/>

2
webapp/app/queries/translate-text-project.ts
View File

@ -4,7 +4,7 @@ export default gql`
query TranslateTextProject(
$projectId: ID!
$text: String!
$sourceLanguageSlug: String!
$sourceLanguageSlug: String
$targetLanguageSlug: String!
) {
viewer {

22
webapp/tsconfig.json
View File

@ -21,12 +21,24 @@
"inlineSources": true,
"baseUrl": ".",
"module": "es6",
"lib": ["dom", "es2020", "esnext.asynciterable"],
"lib": [
"dom",
"es2021",
"esnext.asynciterable"
],
"paths": {
"accent-webapp/tests/*": ["tests/*"],
"accent-webapp/*": ["app/*"],
"fetch": ["node_modules/ember-fetch"],
"*": ["types/*"]
"accent-webapp/tests/*": [
"tests/*"
],
"accent-webapp/*": [
"app/*"
],
"fetch": [
"node_modules/ember-fetch"
],
"*": [
"types/*"
]
}
},
"include": [