diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.ex index f5d5fe678..cb957576a 100644 --- a/lib/plausible/event/clickhouse_schema.ex +++ b/lib/plausible/event/clickhouse_schema.ex @@ -33,6 +33,7 @@ defmodule Plausible.ClickhouseEvent do field :"meta.key", {:array, :string}, default: [] field :"meta.value", {:array, :string}, default: [] + field :transferred_from, :string, default: "" end def new(attrs) do diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index ccf653049..00b0b1f31 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -37,6 +37,8 @@ defmodule Plausible.ClickhouseSession do field :browser, :string field :browser_version, :string field :timestamp, :naive_datetime + + field :transferred_from, :string, default: "" end def random_uint64() do diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex index f2d2938d5..32b6034c7 100644 --- a/lib/plausible/site/admin.ex +++ b/lib/plausible/site/admin.ex @@ -1,5 +1,6 @@ defmodule Plausible.SiteAdmin do use Plausible.Repo + import Ecto.Query def search_fields(_schema) do [ @@ -31,6 +32,22 @@ defmodule Plausible.SiteAdmin do ] end + def list_actions(_conn) do + [ + transfer_data: %{ + name: "Transfer data", + inputs: [ + %{name: "domain", title: "to domain", default: nil} + ], + action: fn _conn, sites, params -> transfer_data(sites, params) end + } + ] + end + + defp format_date(date) do + Timex.format!(date, "{Mshort} {D}, {YYYY}") + end + defp get_owner_email(site) do Enum.find(site.memberships, fn m -> m.role == :owner end).user.email end @@ -40,7 +57,67 @@ defmodule Plausible.SiteAdmin do Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ") end - defp format_date(date) do - Timex.format!(date, "{Mshort} {D}, {YYYY}") + def transfer_data([site], params) do + from_domain = site.domain + to_domain = params["domain"] + + if to_domain && domain_exists?(to_domain) do + event_q = event_transfer_query(from_domain, to_domain) + {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q) + + session_q = session_transfer_query(from_domain, to_domain) + {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q) + + :ok + else + {:error, "Cannot transfer to non-existing domain"} + end + end + + def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"} + + defp domain_exists?(domain) do + Repo.exists?(from s in Plausible.Site, where: s.domain == ^domain) + end + + def session_transfer_query(from_domain, to_domain) do + fields = get_struct_fields(Plausible.ClickhouseSession) + + "INSERT INTO sessions (" <> + stringify_fields(fields) <> + ") SELECT " <> + stringify_fields(fields, to_domain, from_domain) <> + " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')" + end + + def event_transfer_query(from_domain, to_domain) do + fields = get_struct_fields(Plausible.ClickhouseEvent) + + "INSERT INTO events (" <> + stringify_fields(fields) <> + ") SELECT " <> + stringify_fields(fields, to_domain, from_domain) <> + " FROM (SELECT * FROM events WHERE domain='#{from_domain}')" + end + + def get_struct_fields(module) do + module.__struct__() + |> Map.drop([:__meta__, :__struct__]) + |> Map.keys() + |> Enum.map(&Atom.to_string/1) + |> Enum.sort() + end + + defp stringify_fields(fields), do: Enum.join(fields, ", ") + + defp stringify_fields(fields, domain_value, transferred_from_value) do + Enum.map(fields, fn field -> + case field do + "domain" -> "'#{domain_value}' as domain" + "transferred_from" -> "'#{transferred_from_value}' as transferred_from" + _ -> field + end + end) + |> stringify_fields() end end diff --git a/mix.lock b/mix.lock index 4ba1ece28..a556a16ed 100644 --- a/mix.lock +++ b/mix.lock @@ -10,7 +10,7 @@ "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"}, "chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"}, "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []}, - "clickhousex": {:git, "https://github.com/plausible/clickhousex", "6405ac09b4fa103644bb4fe7fc0509fb48497927", []}, + "clickhousex": {:git, "https://github.com/plausible/clickhousex", "f030155234ad045a08a9fdc263f471dd4cfea350", []}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"}, diff --git a/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs new file mode 100644 index 000000000..0e72288c7 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs @@ -0,0 +1,13 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddTransferredFrom do + use Ecto.Migration + + def change do + alter table(:events) do + add(:transferred_from, :string) + end + + alter table(:sessions) do + add(:transferred_from, :string) + end + end +end diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs new file mode 100644 index 000000000..15b916c94 --- /dev/null +++ b/test/plausible/site/admin_test.exs @@ -0,0 +1,82 @@ +defmodule Plausible.SiteAdminTest do + use Plausible.DataCase + import Plausible.TestUtils + alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession} + + test "event and session structs remain the same after transfer" do + from_site = insert(:site) + to_site = insert(:site) + + populate_stats(from_site, [build(:pageview)]) + + event_before = get_event_by_domain(from_site.domain) + session_before = get_session_by_domain(from_site.domain) + + SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) + + event_after = get_event_by_domain(to_site.domain) + session_after = get_session_by_domain(to_site.domain) + + assert event_before == %ClickhouseEvent{event_after | transferred_from: ""} + assert session_before == %ClickhouseSession{session_after | transferred_from: ""} + assert event_after.transferred_from == from_site.domain + assert session_after.transferred_from == from_site.domain + end + + test "transfers all events and sessions" do + from_site = insert(:site) + to_site = insert(:site) + + populate_stats(from_site, [ + build(:pageview, user_id: 123), + build(:event, name: "Signup", user_id: 123), + build(:pageview, user_id: 456), + build(:event, name: "Signup", user_id: 789) + ]) + + SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) + + transferred_events = + ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain) + + transferred_sessions = + ClickhouseRepo.all( + from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain + ) + + assert length(transferred_events) == 4 + assert length(transferred_sessions) == 3 + end + + test "session_transfer_query" do + actual = SiteAdmin.session_transfer_query("from.com", "to.com") + + expected = + "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + + assert actual == expected + end + + test "event_transfer_query" do + actual = SiteAdmin.event_transfer_query("from.com", "to.com") + + expected = + "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')" + + assert actual == expected + end + + defp get_event_by_domain(domain) do + q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain + + Plausible.ClickhouseRepo.one!(q) + |> Map.drop([:__meta__, :domain]) + end + + defp get_session_by_domain(domain) do + q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain + + Plausible.ClickhouseRepo.one!(q) + |> Map.drop([:__meta__, :domain]) + end +end diff --git a/test/plausible/sites_test.exs b/test/plausible/site/sites_test.exs similarity index 100% rename from test/plausible/sites_test.exs rename to test/plausible/site/sites_test.exs