Crm transfer data (#1749)

* pull from master

* added query generation by struct fields

* ready, improved tests

* fixed a naming mistake

lib/plausible/event/clickhouse_schema.ex

@@ -33,6 +33,7 @@ defmodule Plausible.ClickhouseEvent do
 
     field :"meta.key", {:array, :string}, default: []
     field :"meta.value", {:array, :string}, default: []
+    field :transferred_from, :string, default: ""
   end
 
   def new(attrs) do

lib/plausible/session/clickhouse_schema.ex

@@ -37,6 +37,8 @@ defmodule Plausible.ClickhouseSession do
     field :browser, :string
     field :browser_version, :string
     field :timestamp, :naive_datetime
+
+    field :transferred_from, :string, default: ""
   end
 
   def random_uint64() do

lib/plausible/site/admin.ex

@@ -1,5 +1,6 @@
 defmodule Plausible.SiteAdmin do
   use Plausible.Repo
+  import Ecto.Query
 
   def search_fields(_schema) do
     [
@@ -31,6 +32,22 @@ defmodule Plausible.SiteAdmin do
     ]
   end
 
+  def list_actions(_conn) do
+    [
+      transfer_data: %{
+        name: "Transfer data",
+        inputs: [
+          %{name: "domain", title: "to domain", default: nil}
+        ],
+        action: fn _conn, sites, params -> transfer_data(sites, params) end
+      }
+    ]
+  end
+
+  defp format_date(date) do
+    Timex.format!(date, "{Mshort} {D}, {YYYY}")
+  end
+
   defp get_owner_email(site) do
     Enum.find(site.memberships, fn m -> m.role == :owner end).user.email
   end
@@ -40,7 +57,67 @@ defmodule Plausible.SiteAdmin do
     Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")
   end
 
-  defp format_date(date) do
-    Timex.format!(date, "{Mshort} {D}, {YYYY}")
+  def transfer_data([site], params) do
+    from_domain = site.domain
+    to_domain = params["domain"]
+
+    if to_domain && domain_exists?(to_domain) do
+      event_q = event_transfer_query(from_domain, to_domain)
+      {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)
+
+      session_q = session_transfer_query(from_domain, to_domain)
+      {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)
+
+      :ok
+    else
+      {:error, "Cannot transfer to non-existing domain"}
+    end
+  end
+
+  def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}
+
+  defp domain_exists?(domain) do
+    Repo.exists?(from s in Plausible.Site, where: s.domain == ^domain)
+  end
+
+  def session_transfer_query(from_domain, to_domain) do
+    fields = get_struct_fields(Plausible.ClickhouseSession)
+
+    "INSERT INTO sessions (" <>
+      stringify_fields(fields) <>
+      ") SELECT " <>
+      stringify_fields(fields, to_domain, from_domain) <>
+      " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')"
+  end
+
+  def event_transfer_query(from_domain, to_domain) do
+    fields = get_struct_fields(Plausible.ClickhouseEvent)
+
+    "INSERT INTO events (" <>
+      stringify_fields(fields) <>
+      ") SELECT " <>
+      stringify_fields(fields, to_domain, from_domain) <>
+      " FROM (SELECT * FROM events WHERE domain='#{from_domain}')"
+  end
+
+  def get_struct_fields(module) do
+    module.__struct__()
+    |> Map.drop([:__meta__, :__struct__])
+    |> Map.keys()
+    |> Enum.map(&Atom.to_string/1)
+    |> Enum.sort()
+  end
+
+  defp stringify_fields(fields), do: Enum.join(fields, ", ")
+
+  defp stringify_fields(fields, domain_value, transferred_from_value) do
+    Enum.map(fields, fn field ->
+      case field do
+        "domain" -> "'#{domain_value}' as domain"
+        "transferred_from" -> "'#{transferred_from_value}' as transferred_from"
+        _ -> field
+      end
+    end)
+    |> stringify_fields()
   end
 end

mix.lock

@@ -10,7 +10,7 @@
   "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
   "chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"},
   "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []},
-  "clickhousex": {:git, "https://github.com/plausible/clickhousex", "6405ac09b4fa103644bb4fe7fc0509fb48497927", []},
+  "clickhousex": {:git, "https://github.com/plausible/clickhousex", "f030155234ad045a08a9fdc263f471dd4cfea350", []},
   "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},

priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs

@@ -0,0 +1,13 @@
+defmodule Plausible.ClickhouseRepo.Migrations.AddTransferredFrom do
+  use Ecto.Migration
+
+  def change do
+    alter table(:events) do
+      add(:transferred_from, :string)
+    end
+
+    alter table(:sessions) do
+      add(:transferred_from, :string)
+    end
+  end
+end

test/plausible/site/admin_test.exs

@@ -0,0 +1,82 @@
+defmodule Plausible.SiteAdminTest do
+  use Plausible.DataCase
+  import Plausible.TestUtils
+  alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
+
+  test "event and session structs remain the same after transfer" do
+    from_site = insert(:site)
+    to_site = insert(:site)
+
+    populate_stats(from_site, [build(:pageview)])
+
+    event_before = get_event_by_domain(from_site.domain)
+    session_before = get_session_by_domain(from_site.domain)
+
+    SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+    event_after = get_event_by_domain(to_site.domain)
+    session_after = get_session_by_domain(to_site.domain)
+
+    assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+    assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
+    assert event_after.transferred_from == from_site.domain
+    assert session_after.transferred_from == from_site.domain
+  end
+
+  test "transfers all events and sessions" do
+    from_site = insert(:site)
+    to_site = insert(:site)
+
+    populate_stats(from_site, [
+      build(:pageview, user_id: 123),
+      build(:event, name: "Signup", user_id: 123),
+      build(:pageview, user_id: 456),
+      build(:event, name: "Signup", user_id: 789)
+    ])
+
+    SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+    transferred_events =
+      ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)
+
+    transferred_sessions =
+      ClickhouseRepo.all(
+        from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
+      )
+
+    assert length(transferred_events) == 4
+    assert length(transferred_sessions) == 3
+  end
+
+  test "session_transfer_query" do
+    actual = SiteAdmin.session_transfer_query("from.com", "to.com")
+
+    expected =
+      "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+
+    assert actual == expected
+  end
+
+  test "event_transfer_query" do
+    actual = SiteAdmin.event_transfer_query("from.com", "to.com")
+
+    expected =
+      "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
+
+    assert actual == expected
+  end
+
+  defp get_event_by_domain(domain) do
+    q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
+
+    Plausible.ClickhouseRepo.one!(q)
+    |> Map.drop([:__meta__, :domain])
+  end
+
+  defp get_session_by_domain(domain) do
+    q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain
+
+    Plausible.ClickhouseRepo.one!(q)
+    |> Map.drop([:__meta__, :domain])
+  end
+end