Add stats_start_date field

lib/plausible/site/admin.ex

@@ -57,17 +57,22 @@ defmodule Plausible.SiteAdmin do
     Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")
   end
 
-  def transfer_data([site], params) do
-    from_domain = site.domain
-    to_domain = params["domain"]
+  def transfer_data([from_site], params) do
+    to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
 
-    if to_domain && domain_exists?(to_domain) do
-      event_q = event_transfer_query(from_domain, to_domain)
+    if to_site do
+      event_q = event_transfer_query(from_site.domain, to_site.domain)
       {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)
 
-      session_q = session_transfer_query(from_domain, to_domain)
+      session_q = session_transfer_query(from_site.domain, to_site.domain)
       {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)
 
+      start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
+
+      {:ok, _} =
+        Plausible.Site.set_stats_start_date(to_site, start_date)
+        |> Repo.update()
+
       :ok
     else
       {:error, "Cannot transfer to non-existing domain"}

lib/plausible/site/schema.ex

@@ -22,6 +22,7 @@ defmodule Plausible.Site do
     field :public, :boolean
     field :locked, :boolean
     field :has_stats, :boolean
+    field :stats_start_date, :date
 
     embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
 
@@ -63,6 +64,10 @@ defmodule Plausible.Site do
     change(site, has_stats: has_stats_val)
   end
 
+  def set_stats_start_date(site, val) do
+    change(site, stats_start_date: val)
+  end
+
   def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
     change(site,
       imported_data: %{
@@ -75,7 +80,10 @@ defmodule Plausible.Site do
   end
 
   def import_success(site) do
-    change(site, imported_data: %{status: "ok"})
+    change(site,
+      stats_start_date: site.imported_data.start_date,
+      imported_data: %{status: "ok"}
+    )
   end
 
   def import_failure(site) do

lib/plausible/sites.ex

@@ -38,6 +38,21 @@ defmodule Plausible.Sites do
     end
   end
 
+  def stats_start_date(site) do
+    if site.stats_start_date do
+      site.stats_start_date
+    else
+      start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+
+      if start_date do
+        Plausible.Site.set_stats_start_date(site, start_date)
+        |> Repo.update()
+
+        start_date
+      end
+    end
+  end
+
   def has_stats?(site) do
     if site.has_stats do
       true

lib/plausible/stats/clickhouse.ex

@@ -16,7 +16,7 @@ defmodule Plausible.Stats.Clickhouse do
     case datetime do
       # no stats for this domain yet
       ~N[1970-01-01 00:00:00] ->
-        Timex.today(site.timezone)
+        nil
 
       _ ->
         Timex.Timezone.convert(datetime, "UTC")

lib/plausible_web/controllers/site_controller.ex

@@ -701,7 +701,10 @@ defmodule PlausibleWeb.SiteController do
     site = conn.assigns[:site]
 
     start_date = Plausible.Google.Api.get_analytics_start_date(view_id, access_token)
-    end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+
+    end_date =
+      Plausible.Stats.Clickhouse.pageview_start_date_local(site) || Timex.today(site.timezone)
+
     {:ok, view_ids} = Plausible.Google.Api.get_analytics_view_ids(access_token)
 
     conn

priv/repo/migrations/20220405124819_add_stats_start_date.exs

@@ -0,0 +1,9 @@
+defmodule Plausible.Repo.Migrations.AddStatsStartDate do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sites) do
+      add :stats_start_date, :date
+    end
+  end
+end

test/plausible/site/admin_test.exs

@@ -3,67 +3,82 @@ defmodule Plausible.SiteAdminTest do
   import Plausible.TestUtils
   alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
 
-  test "event and session structs remain the same after transfer" do
-    from_site = insert(:site)
-    to_site = insert(:site)
+  describe "transfer_data" do
+    test "event and session structs remain the same after transfer" do
+      from_site = insert(:site)
+      to_site = insert(:site)
 
-    populate_stats(from_site, [build(:pageview)])
+      populate_stats(from_site, [build(:pageview)])
 
-    event_before = get_event_by_domain(from_site.domain)
-    session_before = get_session_by_domain(from_site.domain)
+      event_before = get_event_by_domain(from_site.domain)
+      session_before = get_session_by_domain(from_site.domain)
 
-    SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+      SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
 
-    event_after = get_event_by_domain(to_site.domain)
-    session_after = get_session_by_domain(to_site.domain)
+      event_after = get_event_by_domain(to_site.domain)
+      session_after = get_session_by_domain(to_site.domain)
 
-    assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
-    assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
-    assert event_after.transferred_from == from_site.domain
-    assert session_after.transferred_from == from_site.domain
-  end
+      assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+      assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
+      assert event_after.transferred_from == from_site.domain
+      assert session_after.transferred_from == from_site.domain
+    end
 
-  test "transfers all events and sessions" do
-    from_site = insert(:site)
-    to_site = insert(:site)
+    test "transfers all events and sessions" do
+      from_site = insert(:site)
+      to_site = insert(:site)
 
-    populate_stats(from_site, [
-      build(:pageview, user_id: 123),
-      build(:event, name: "Signup", user_id: 123),
-      build(:pageview, user_id: 456),
-      build(:event, name: "Signup", user_id: 789)
-    ])
+      populate_stats(from_site, [
+        build(:pageview, user_id: 123),
+        build(:event, name: "Signup", user_id: 123),
+        build(:pageview, user_id: 456),
+        build(:event, name: "Signup", user_id: 789)
+      ])
 
-    SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+      SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
 
-    transferred_events =
-      ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)
+      transferred_events =
+        ClickhouseRepo.all(
+          from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
+        )
 
-    transferred_sessions =
-      ClickhouseRepo.all(
-        from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
-      )
+      transferred_sessions =
+        ClickhouseRepo.all(
+          from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
+        )
 
-    assert length(transferred_events) == 4
-    assert length(transferred_sessions) == 3
-  end
+      assert length(transferred_events) == 4
+      assert length(transferred_sessions) == 3
+    end
 
-  test "session_transfer_query" do
-    actual = SiteAdmin.session_transfer_query("from.com", "to.com")
+    test "updates stats_start_date on site record" do
+      from_site = insert(:site)
+      to_site = insert(:site)
 
-    expected =
-      "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+      populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
 
-    assert actual == expected
-  end
+      SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
 
-  test "event_transfer_query" do
-    actual = SiteAdmin.event_transfer_query("from.com", "to.com")
+      assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
+    end
 
-    expected =
-      "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
+    test "session_transfer_query" do
+      actual = SiteAdmin.session_transfer_query("from.com", "to.com")
 
-    assert actual == expected
+      expected =
+        "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+
+      assert actual == expected
+    end
+
+    test "event_transfer_query" do
+      actual = SiteAdmin.event_transfer_query("from.com", "to.com")
+
+      expected =
+        "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
+
+      assert actual == expected
+    end
   end
 
   defp get_event_by_domain(domain) do

test/plausible/site/sites_test.exs

@@ -19,6 +19,37 @@ defmodule Plausible.SitesTest do
     end
   end
 
+  describe "stats_start_date" do
+    test "is nil if site has no stats" do
+      site = insert(:site)
+
+      assert Sites.stats_start_date(site) == nil
+    end
+
+    test "is date if first pageview if site does have stats" do
+      site = insert(:site)
+
+      populate_stats(site, [
+        build(:pageview)
+      ])
+
+      assert Sites.stats_start_date(site) == Timex.today(site.timezone)
+    end
+
+    test "memoizes value of start date" do
+      site = insert(:site)
+
+      assert site.stats_start_date == nil
+
+      populate_stats(site, [
+        build(:pageview)
+      ])
+
+      assert Sites.stats_start_date(site) == Timex.today(site.timezone)
+      assert Repo.reload!(site).stats_start_date == Timex.today(site.timezone)
+    end
+  end
+
   describe "has_stats?" do
     test "is false if site has no stats" do
       site = insert(:site)

test/workers/import_google_analytics_test.exs

@@ -6,6 +6,7 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
   alias Plausible.Workers.ImportGoogleAnalytics
 
   @imported_data %Plausible.Site.ImportedData{
+    start_date: Timex.today() |> Timex.shift(days: -7),
     end_date: Timex.today(),
     source: "Google Analytics",
     status: "importing"
@@ -40,6 +41,35 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
     assert Repo.reload!(site).imported_data.status == "ok"
   end
 
+  test "updates the stats_start_date field for the site after succesful import" do
+    user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+    site = insert(:site, members: [user], imported_data: @imported_data)
+
+    api_stub =
+      stub(Plausible.Google.Api, :import_analytics, fn _site,
+                                                       _view_id,
+                                                       _start_date,
+                                                       _end_date,
+                                                       _access_token ->
+        {:ok, nil}
+      end)
+
+    ImportGoogleAnalytics.perform(
+      %Oban.Job{
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "view_id",
+          "start_date" => "2020-01-01",
+          "end_date" => "2022-01-01",
+          "access_token" => "token"
+        }
+      },
+      api_stub
+    )
+
+    assert Repo.reload!(site).stats_start_date == @imported_data.start_date
+  end
+
   test "sends email to owner after succesful import" do
     user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
     site = insert(:site, members: [user], imported_data: @imported_data)