diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex index 32b6034c7..4c8fc863b 100644 --- a/lib/plausible/site/admin.ex +++ b/lib/plausible/site/admin.ex @@ -57,17 +57,22 @@ defmodule Plausible.SiteAdmin do Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ") end - def transfer_data([site], params) do - from_domain = site.domain - to_domain = params["domain"] + def transfer_data([from_site], params) do + to_site = Repo.get_by(Plausible.Site, domain: params["domain"]) - if to_domain && domain_exists?(to_domain) do - event_q = event_transfer_query(from_domain, to_domain) + if to_site do + event_q = event_transfer_query(from_site.domain, to_site.domain) {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q) - session_q = session_transfer_query(from_domain, to_domain) + session_q = session_transfer_query(from_site.domain, to_site.domain) {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q) + start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site) + + {:ok, _} = + Plausible.Site.set_stats_start_date(to_site, start_date) + |> Repo.update() + :ok else {:error, "Cannot transfer to non-existing domain"} diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 127dffc01..1a4afe8d6 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -22,6 +22,7 @@ defmodule Plausible.Site do field :public, :boolean field :locked, :boolean field :has_stats, :boolean + field :stats_start_date, :date embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update @@ -63,6 +64,10 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end + def set_stats_start_date(site, val) do + change(site, stats_start_date: val) + end + def start_import(site, start_date, end_date, imported_source, status \\ "importing") do change(site, imported_data: %{ @@ -75,7 +80,10 @@ defmodule Plausible.Site do end def import_success(site) do - change(site, imported_data: %{status: "ok"}) + change(site, + stats_start_date: site.imported_data.start_date, + imported_data: %{status: "ok"} + ) end def import_failure(site) do diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 25bba9c84..27ba78f04 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -38,6 +38,21 @@ defmodule Plausible.Sites do end end + def stats_start_date(site) do + if site.stats_start_date do + site.stats_start_date + else + start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) + + if start_date do + Plausible.Site.set_stats_start_date(site, start_date) + |> Repo.update() + + start_date + end + end + end + def has_stats?(site) do if site.has_stats do true diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 7dc3705cf..3d8ce1d63 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -16,7 +16,7 @@ defmodule Plausible.Stats.Clickhouse do case datetime do # no stats for this domain yet ~N[1970-01-01 00:00:00] -> - Timex.today(site.timezone) + nil _ -> Timex.Timezone.convert(datetime, "UTC") diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 728e835f4..53f70ffa8 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -701,7 +701,10 @@ defmodule PlausibleWeb.SiteController do site = conn.assigns[:site] start_date = Plausible.Google.Api.get_analytics_start_date(view_id, access_token) - end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) + + end_date = + Plausible.Stats.Clickhouse.pageview_start_date_local(site) || Timex.today(site.timezone) + {:ok, view_ids} = Plausible.Google.Api.get_analytics_view_ids(access_token) conn diff --git a/priv/repo/migrations/20220405124819_add_stats_start_date.exs b/priv/repo/migrations/20220405124819_add_stats_start_date.exs new file mode 100644 index 000000000..250d1f9a9 --- /dev/null +++ b/priv/repo/migrations/20220405124819_add_stats_start_date.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddStatsStartDate do + use Ecto.Migration + + def change do + alter table(:sites) do + add :stats_start_date, :date + end + end +end diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs index 15b916c94..f5cd92538 100644 --- a/test/plausible/site/admin_test.exs +++ b/test/plausible/site/admin_test.exs @@ -3,67 +3,82 @@ defmodule Plausible.SiteAdminTest do import Plausible.TestUtils alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession} - test "event and session structs remain the same after transfer" do - from_site = insert(:site) - to_site = insert(:site) + describe "transfer_data" do + test "event and session structs remain the same after transfer" do + from_site = insert(:site) + to_site = insert(:site) - populate_stats(from_site, [build(:pageview)]) + populate_stats(from_site, [build(:pageview)]) - event_before = get_event_by_domain(from_site.domain) - session_before = get_session_by_domain(from_site.domain) + event_before = get_event_by_domain(from_site.domain) + session_before = get_session_by_domain(from_site.domain) - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) + SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - event_after = get_event_by_domain(to_site.domain) - session_after = get_session_by_domain(to_site.domain) + event_after = get_event_by_domain(to_site.domain) + session_after = get_session_by_domain(to_site.domain) - assert event_before == %ClickhouseEvent{event_after | transferred_from: ""} - assert session_before == %ClickhouseSession{session_after | transferred_from: ""} - assert event_after.transferred_from == from_site.domain - assert session_after.transferred_from == from_site.domain - end + assert event_before == %ClickhouseEvent{event_after | transferred_from: ""} + assert session_before == %ClickhouseSession{session_after | transferred_from: ""} + assert event_after.transferred_from == from_site.domain + assert session_after.transferred_from == from_site.domain + end - test "transfers all events and sessions" do - from_site = insert(:site) - to_site = insert(:site) + test "transfers all events and sessions" do + from_site = insert(:site) + to_site = insert(:site) - populate_stats(from_site, [ - build(:pageview, user_id: 123), - build(:event, name: "Signup", user_id: 123), - build(:pageview, user_id: 456), - build(:event, name: "Signup", user_id: 789) - ]) + populate_stats(from_site, [ + build(:pageview, user_id: 123), + build(:event, name: "Signup", user_id: 123), + build(:pageview, user_id: 456), + build(:event, name: "Signup", user_id: 789) + ]) - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) + SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - transferred_events = - ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain) + transferred_events = + ClickhouseRepo.all( + from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain + ) - transferred_sessions = - ClickhouseRepo.all( - from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain - ) + transferred_sessions = + ClickhouseRepo.all( + from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain + ) - assert length(transferred_events) == 4 - assert length(transferred_sessions) == 3 - end + assert length(transferred_events) == 4 + assert length(transferred_sessions) == 3 + end - test "session_transfer_query" do - actual = SiteAdmin.session_transfer_query("from.com", "to.com") + test "updates stats_start_date on site record" do + from_site = insert(:site) + to_site = insert(:site) - expected = - "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])]) - assert actual == expected - end + SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - test "event_transfer_query" do - actual = SiteAdmin.event_transfer_query("from.com", "to.com") + assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01] + end - expected = - "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')" + test "session_transfer_query" do + actual = SiteAdmin.session_transfer_query("from.com", "to.com") - assert actual == expected + expected = + "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + + assert actual == expected + end + + test "event_transfer_query" do + actual = SiteAdmin.event_transfer_query("from.com", "to.com") + + expected = + "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')" + + assert actual == expected + end end defp get_event_by_domain(domain) do diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs index 077c694d4..492bfa44a 100644 --- a/test/plausible/site/sites_test.exs +++ b/test/plausible/site/sites_test.exs @@ -19,6 +19,37 @@ defmodule Plausible.SitesTest do end end + describe "stats_start_date" do + test "is nil if site has no stats" do + site = insert(:site) + + assert Sites.stats_start_date(site) == nil + end + + test "is date if first pageview if site does have stats" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview) + ]) + + assert Sites.stats_start_date(site) == Timex.today(site.timezone) + end + + test "memoizes value of start date" do + site = insert(:site) + + assert site.stats_start_date == nil + + populate_stats(site, [ + build(:pageview) + ]) + + assert Sites.stats_start_date(site) == Timex.today(site.timezone) + assert Repo.reload!(site).stats_start_date == Timex.today(site.timezone) + end + end + describe "has_stats?" do test "is false if site has no stats" do site = insert(:site) diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exs index 5caa6d36e..1852945a4 100644 --- a/test/workers/import_google_analytics_test.exs +++ b/test/workers/import_google_analytics_test.exs @@ -6,6 +6,7 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do alias Plausible.Workers.ImportGoogleAnalytics @imported_data %Plausible.Site.ImportedData{ + start_date: Timex.today() |> Timex.shift(days: -7), end_date: Timex.today(), source: "Google Analytics", status: "importing" @@ -40,6 +41,35 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do assert Repo.reload!(site).imported_data.status == "ok" end + test "updates the stats_start_date field for the site after succesful import" do + user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) + site = insert(:site, members: [user], imported_data: @imported_data) + + api_stub = + stub(Plausible.Google.Api, :import_analytics, fn _site, + _view_id, + _start_date, + _end_date, + _access_token -> + {:ok, nil} + end) + + ImportGoogleAnalytics.perform( + %Oban.Job{ + args: %{ + "site_id" => site.id, + "view_id" => "view_id", + "start_date" => "2020-01-01", + "end_date" => "2022-01-01", + "access_token" => "token" + } + }, + api_stub + ) + + assert Repo.reload!(site).stats_start_date == @imported_data.start_date + end + test "sends email to owner after succesful import" do user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) site = insert(:site, members: [user], imported_data: @imported_data)