International Domain Names (IDN) Support (#2034)

* Accept letters from non-Latin alphabets in domain names

* Replace static URLs with Router functions in settings_visibility

* Beautify dashboard URL in visibility tab

* Add IDN support to CHANGELOG

CHANGELOG.md

@@ -38,6 +38,7 @@ All notable changes to this project will be documented in this file.
 - Alert outgrown enterprise users of their usage plausible/analytics#2197
 - Manually lock and unlock enterprise users plausible/analytics#2197
 - ARM64 support for docker images plausible/analytics#2103
+- Add support for international domain names (IDNs) plausible/analytics#2034
 
 ### Fixed
 - Hash part of the URL can now be used when excluding pages with `script.exclusions.hash.js`.

lib/plausible/site/schema.ex

@@ -43,14 +43,15 @@ defmodule Plausible.Site do
     site
     |> cast(attrs, [:domain, :timezone])
     |> validate_required([:domain, :timezone])
-    |> validate_format(:domain, ~r/^[a-zA-Z0-9\-\.\/\:]*$/,
+    |> clean_domain()
+    |> validate_format(:domain, ~r/^[-\.\\\/:\p{L}\d]*$/u,
       message: "only letters, numbers, slashes and period allowed"
     )
+    |> validate_domain_reserved_characters()
     |> unique_constraint(:domain,
       message:
         "This domain has already been taken. Perhaps one of your team members registered it? If that's not the case, please contact support@plausible.io"
     )
-    |> clean_domain
   end
 
   def make_public(site) do
@@ -114,4 +115,20 @@ defmodule Plausible.Site do
       domain: clean_domain
     })
   end
+
+  # https://tools.ietf.org/html/rfc3986#section-2.2
+  @uri_reserved_chars ~w(: ? # [ ] @ ! $ & ' \( \) * + , ; =)
+  defp validate_domain_reserved_characters(changeset) do
+    domain = get_field(changeset, :domain) || ""
+
+    if String.contains?(domain, @uri_reserved_chars) do
+      add_error(
+        changeset,
+        :domain,
+        "must not contain URI reserved characters #{@uri_reserved_chars}"
+      )
+    else
+      changeset
+    end
+  end
 end

lib/plausible_web/templates/site/settings_visibility.html.eex

@@ -7,21 +7,25 @@
     <% end %>
   </header>
 
-	<%= if @site.public do %>
+  <%= if @site.public do %>
-		<div class="flex items-center mt-4 space-x-3">
+    <div class="flex items-center mt-4 space-x-3">
-			<%= button(to: "/sites/#{URI.encode_www_form(@site.domain)}/make-private", method: "POST", class: "bg-indigo-600 relative inline-flex flex-shrink-0 h-6 w-11 border-2 border-transparent rounded-full cursor-pointer transition-colors ease-in-out duration-200 focus:outline-none focus:ring") do %>
+      <%= button(to: Routes.site_path(@conn, :make_private, @site.domain), method: "POST", class: "bg-indigo-600 relative inline-flex flex-shrink-0 h-6 w-11 border-2 border-transparent rounded-full cursor-pointer transition-colors ease-in-out duration-200 focus:outline-none focus:ring") do %>
-				<span class="inline-block w-5 h-5 bg-white rounded-full shadow translate-x-5 dark:bg-gray-800 transform transition ease-in-out duration-200"></span>
+        <span class="inline-block w-5 h-5 bg-white rounded-full shadow translate-x-5 dark:bg-gray-800 transform transition ease-in-out duration-200"></span>
-			<% end %>
+      <% end %>
-			<span class="text-sm font-medium text-gray-900 leading-5 dark:text-gray-100">Make stats publicly available on <a href="<%= plausible_url() <> "/" <> URI.encode_www_form(@site.domain)%>" class="text-indigo-500"><%= plausible_url() <> "/" <> URI.encode_www_form(@site.domain)%></a></span>
+      <span class="text-sm font-medium text-gray-900 leading-5 dark:text-gray-100">
-		</div>
+        Stats are publicly available on <%= link(PlausibleWeb.StatsView.pretty_stats_url(@site), to: Routes.stats_path(@conn, :stats, @site.domain, []), class: "text-indigo-500") %>
-	<% else %>
+      </span>
-		<div class="flex items-center mt-4 space-x-3">
+    </div>
-			<%= button(to: "/sites/#{URI.encode_www_form(@site.domain)}/make-public", method: "POST", class: "bg-gray-200 dark:bg-gray-700 relative inline-flex flex-shrink-0 h-6 w-11 border-2 border-transparent rounded-full cursor-pointer transition-colors ease-in-out duration-200 focus:outline-none focus:ring") do %>
+  <% else %>
-				<span class="inline-block w-5 h-5 bg-white rounded-full shadow translate-x-0 dark:bg-gray-800 transform transition ease-in-out duration-200"></span>
+    <div class="flex items-center mt-4 space-x-3">
-			<% end %>
+      <%= button(to: Routes.site_path(@conn, :make_public, @site.domain), method: "POST", class: "bg-gray-200 dark:bg-gray-700 relative inline-flex flex-shrink-0 h-6 w-11 border-2 border-transparent rounded-full cursor-pointer transition-colors ease-in-out duration-200 focus:outline-none focus:ring") do %>
-			<span class="text-sm font-medium text-gray-900 leading-5 dark:text-gray-100">Make stats publicly available on <a href="<%= plausible_url() <> "/" <> URI.encode_www_form(@site.domain)%>" class="text-indigo-500"><%= plausible_url() <> "/" <> URI.encode_www_form(@site.domain)%></a></span>
+        <span class="inline-block w-5 h-5 bg-white rounded-full shadow translate-x-0 dark:bg-gray-800 transform transition ease-in-out duration-200"></span>
-		</div>
+      <% end %>
-	<% end %>
+      <span class="text-sm font-medium text-gray-900 leading-5 dark:text-gray-100">
+        Make stats publicly available on <%= link(PlausibleWeb.StatsView.pretty_stats_url(@site), to: Routes.stats_path(@conn, :stats, @site.domain, []), class: "text-indigo-500") %>
+      </span>
+    </div>
+  <% end %>
 </div>
 
 <div class="px-4 py-6 bg-white shadow dark:bg-gray-800 sm:rounded-md sm:overflow-hidden sm:p-6">
@@ -50,17 +54,18 @@
             <svg class="w-4 h-4" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg"><path d="M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z"></path><path d="M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z"></path></svg>
             <span class="ml-1">Copy</span>
           </button>
-          <%= link(to: "/sites/#{URI.encode_www_form(@site.domain)}/shared-links/#{link.slug}/edit", class: "px-4 py-2 inline-flex items-center text-indigo-800 bg-gray-200 border-r border-gray-300 rounded-none dark:bg-gray-850 dark:text-indigo-500 dark:border-gray-500 hover:bg-gray-300 dark:hover:bg-gray-825") do %>
+
+          <%= link(to: Routes.site_path(@conn, :edit_shared_link, @site.domain, link.slug), class: "px-4 py-2 inline-flex items-center text-indigo-800 bg-gray-200 border-r border-gray-300 rounded-none dark:bg-gray-850 dark:text-indigo-500 dark:border-gray-500 hover:bg-gray-300 dark:hover:bg-gray-825") do %>
             <svg class="w-4 h-4" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg"><path d="M17.414 2.586a2 2 0 00-2.828 0L7 10.172V13h2.828l7.586-7.586a2 2 0 000-2.828z"></path><path fill-rule="evenodd" d="M2 6a2 2 0 012-2h4a1 1 0 010 2H4v10h10v-4a1 1 0 112 0v4a2 2 0 01-2 2H4a2 2 0 01-2-2V6z" clip-rule="evenodd"></path></svg>
           <% end %>
-          <%= button(to: "/sites/#{URI.encode_www_form(@site.domain)}/shared-links/#{link.slug}", method: :delete, class: "py-2 px-4 inline-flex items-center bg-gray-200 dark:bg-gray-850 text-red-600 dark:text-red-500 rounded-l-none hover:bg-gray-300 dark:hover:bg-gray-825", data: [confirm: "Are you sure you want to delete this shared link? The stats will not be accessible with this link anymore."]) do %>
+          <%= button(to: Routes.site_path(@conn, :delete_shared_link, @site.domain, link.slug), method: :delete, class: "py-2 px-4 inline-flex items-center bg-gray-200 dark:bg-gray-850 text-red-600 dark:text-red-500 rounded-l-none hover:bg-gray-300 dark:hover:bg-gray-825", data: [confirm: "Are you sure you want to delete this shared link? The stats will not be accessible with this link anymore."]) do %>
             <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"></path></svg>
           <% end %>
         </div>
       </div>
     <% end %>
 
-    <%= link("+ New link", to: "/sites/#{URI.encode_www_form(@site.domain)}/shared-links/new", class: "button mt-4") %>
+    <%= link("+ New link", to: Routes.site_path(@conn, :new_shared_link, @site.domain), class: "button mt-4") %>
   </div>
 </div>
 

lib/plausible_web/views/stats_view.ex

@@ -71,4 +71,30 @@ defmodule PlausibleWeb.StatsView do
 
     count / max * 100
   end
+
+  @doc """
+  Returns a readable stats URL.
+
+  Native Phoenix router functions percent-encode all diacritics, resulting in
+  ugly URLs, e.g. `https://plausible.io/café.com` transforms into
+  `https://plausible.io/caf%C3%A9.com`.
+
+  This function encodes only the slash (`/`) character from the site's domain.
+
+  ## Examples
+
+     iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "user.gittea.io/repo"})
+     "http://localhost:8000/user.gittea.io%2Frepo"
+
+     iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "anakin.test"})
+     "http://localhost:8000/anakin.test"
+
+     iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "café.test"})
+     "http://localhost:8000/café.test"
+
+  """
+  def pretty_stats_url(%Plausible.Site{domain: domain}) when is_binary(domain) do
+    pretty_domain = String.replace(domain, "/", "%2F")
+    "#{plausible_url()}/#{pretty_domain}"
+  end
 end

test/plausible_web/controllers/api/external_sites_controller_test.exs

@@ -44,6 +44,24 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
              }
     end
 
+    test "accepts international domain names", %{conn: conn} do
+      ["müllers-café.test", "音乐.cn", "до.101домен.рф/pages"]
+      |> Enum.each(fn idn_domain ->
+        conn = post(conn, "/api/v1/sites", %{"domain" => idn_domain})
+        assert %{"domain" => ^idn_domain} = json_response(conn, 200)
+      end)
+    end
+
+    test "validates uri breaking domains", %{conn: conn} do
+      ["quero:café.test", "h&llo.test", "iamnotsur&about?this.com"]
+      |> Enum.each(fn bad_domain ->
+        conn = post(conn, "/api/v1/sites", %{"domain" => bad_domain})
+
+        assert %{"error" => error} = json_response(conn, 400)
+        assert error =~ "domain must not contain URI reserved characters"
+      end)
+    end
+
     test "does not allow creating more sites than the limit", %{conn: conn, user: user} do
       Application.put_env(:plausible, :site_limit, 3)
       insert(:site, members: [user])

test/plausible_web/views/stats_view_test.exs

@@ -1,6 +1,7 @@
-defmodule PlausibleWeb.StatsView.Test do
+defmodule PlausibleWeb.StatsViewTest do
   use PlausibleWeb.ConnCase, async: true
   alias PlausibleWeb.StatsView
+  doctest PlausibleWeb.StatsView
 
   describe "large_number_format" do
     test "numbers under 1000 stay the same" do