Use only root domain in user id generation

lib/plausible_web/controllers/api/external_controller.ex

@@ -195,12 +195,18 @@ defmodule PlausibleWeb.Api.ExternalController do
   defp generate_user_id(conn, domain, hostname, salt) do
     user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
     ip_address = PlausibleWeb.RemoteIp.get(conn)
+    root_domain = get_root_domain(hostname)
 
-    if domain && hostname do
+    if domain && root_domain do
-      SipHash.hash!(salt, user_agent <> ip_address <> domain <> hostname)
+      SipHash.hash!(salt, user_agent <> ip_address <> domain <> root_domain)
     end
   end
 
+  defp get_root_domain(hostname) when is_binary(hostname) do
+    PublicSuffix.registrable_domain(hostname)
+  end
+  defp get_root_domain(hostname),  do: hostname
+
   defp calculate_screen_size(nil), do: nil
   defp calculate_screen_size(width) when width < 576, do: "Mobile"
   defp calculate_screen_size(width) when width < 992, do: "Tablet"

mix.exs

@@ -96,7 +96,8 @@ defmodule Plausible.MixProject do
       {:kaffy, "~> 0.9.0"},
       {:envy, "~> 1.1.1"},
       {:phoenix_pagination, "~> 0.7.0"},
-      {:hammer, "~> 6.0"}
+      {:hammer, "~> 6.0"},
+      {:public_suffix, git: "https://github.com/axelson/publicsuffix-elixir"}
     ]
   end
 

mix.lock

@@ -76,6 +76,7 @@
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
   "postgrex": {:hex, :postgrex, "0.15.9", "46f8fe6f25711aeb861c4d0ae09780facfdf3adbd2fb5594ead61504dd489bda", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "610719103e4cb2223d4ab78f9f0f3e720320eeca6011415ab4137ddef730adee"},
+  "public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "89372422ab8b433de508519ef474e39699fd11ca", []},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
   "ref_inspector": {:hex, :ref_inspector, "1.3.1", "bb0489a4c4299dcd633f2b7a60c41a01f5590789d0b28225a60be484e1fbe777", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "3172eb1b08e5c69966f796e3fe0e691257546fa143a5eb0ecc18a6e39b233854"},
   "sentry": {:hex, :sentry, "8.0.5", "5ca922b9238a50c7258b52f47364b2d545beda5e436c7a43965b34577f1ef61f", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "4972839fdbf52e886d7b3e694c8adf421f764f2fa79036b88fb4742049bd4b7c"},

test/plausible_web/controllers/api/external_controller_test.exs

@@ -12,6 +12,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
     )
   end
 
+  defp get_events(domain) do
+    Plausible.Event.WriteBuffer.flush()
+
+    ClickhouseRepo.all(
+      from e in Plausible.ClickhouseEvent,
+        where: e.domain == ^domain,
+        order_by: [desc: e.timestamp]
+    )
+  end
+
   @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
 
   describe "POST /api/event" do
@@ -357,7 +367,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       assert response(conn, 202) == ""
       assert pageview.referrer_source == ""
     end
-  end
 
     test "screen size is calculated from screen_width", %{conn: conn} do
       params = %{
@@ -692,6 +701,142 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
 
       assert response(conn, 400) == ""
     end
+  end
+
+  describe "user_id generation" do
+    test "with same IP address and user agent, the same user ID is generated", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      [one, two] = get_events("user-id-test-domain.com")
+
+      assert one.user_id == two.user_id
+    end
+
+    test "different IP address results in different user ID", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain-2.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.2")
+      |> post("/api/event", params)
+
+      [one, two] = get_events("user-id-test-domain-2.com")
+
+      assert one.user_id != two.user_id
+    end
+
+    test "different user agent results in different user ID", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain-3.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent <> "!!")
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      [one, two] = get_events("user-id-test-domain-3.com")
+
+      assert one.user_id != two.user_id
+    end
+
+    test "different domain value results in different user ID", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain-4.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", Map.put(params, :domain, "other-domain.com"))
+
+      one = get_event("user-id-test-domain-4.com")
+      two = get_event("other-domain.com")
+
+      assert one.user_id != two.user_id
+    end
+
+    test "different hostname results in different user ID", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain-5.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", Map.put(params, :url, "https://other-domain.com/"))
+
+      [one, two] = get_events("user-id-test-domain-5.com")
+
+      assert one.user_id != two.user_id
+    end
+
+    test "different hostname results in the same user ID when the root domain in the same", %{conn: conn} do
+      params = %{
+        url: "https://user-id-test-domain.com/",
+        domain: "user-id-test-domain-6.com",
+        name: "pageview"
+      }
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", params)
+
+      conn
+      |> put_req_header("user-agent", @user_agent)
+      |> put_req_header("x-forwarded-for", "127.0.0.1")
+      |> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/"))
+
+      [one, two] = get_events("user-id-test-domain-6.com")
+
+      assert one.user_id == two.user_id
+    end
+  end
 
   describe "GET /api/health" do
     test "returns 200 OK", %{conn: conn} do