Optimize usage calculation for large accounts

lib/plausible/billing/billing.ex

@@ -186,14 +186,8 @@ defmodule Plausible.Billing do
   end
 
   def usage_breakdown(user) do
-    sites = Plausible.Sites.owned_by(user)
-
-    Enum.reduce(sites, {0, 0}, fn site, {pageviews, custom_events} ->
-      usage = Plausible.Stats.Clickhouse.usage(site)
-
-      {pageviews + Map.get(usage, "pageviews", 0),
-       custom_events + Map.get(usage, "custom_events", 0)}
-    end)
+    domains = Plausible.Sites.owned_by(user) |> Enum.map(& &1.domain)
+    Plausible.Stats.Clickhouse.usage_breakdown(domains)
   end
 
   @doc """

lib/plausible/stats/clickhouse.ex

@@ -174,21 +174,25 @@ defmodule Plausible.Stats.Clickhouse do
     )
   end
 
-  def usage(site) do
-    q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"})
-    {first_datetime, last_datetime} = utc_boundaries(q, site.timezone)
+  def usage_breakdown(domains) do
+    q = Plausible.Stats.Query.from("UTC", %{"period" => "30d"})
+    {first_datetime, last_datetime} = utc_boundaries(q, "UTC")
 
-    ClickhouseRepo.all(
-      from e in "events",
-        where: e.domain == ^site.domain,
-        where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,
-        group_by: fragment("name"),
-        select: {
-          fragment("if(? = 'pageview', 'pageviews', 'custom_events') as name", e.name),
-          fragment("count(*)")
-        }
-    )
-    |> Enum.into(%{})
+    Enum.chunk_every(domains, 300)
+    |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
+      {chunk_pageviews, chunk_custom_events} =
+        ClickhouseRepo.one(
+          from e in "events",
+            where: e.domain in ^domains,
+            where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,
+            select: {
+              fragment("countIf(? = 'pageview')", e.name),
+              fragment("countIf(? != 'pageview')", e.name)
+            }
+        )
+
+      {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
+    end)
   end
 
   def pageviews_and_visitors(site, query) do

test/plausible/billing/billing_test.exs

@@ -11,11 +11,22 @@ defmodule Plausible.BillingTest do
       assert Billing.usage(user) == 0
     end
 
-    test "counts the total number of events" do
+    test "counts the total number of events from all sites the user owns" do
       user = insert(:user)
-      insert(:site, domain: "test-site.com", members: [user])
+      site1 = insert(:site, members: [user])
+      site2 = insert(:site, members: [user])
 
-      assert Billing.usage(user) == 3
+      populate_stats(site1, [
+        build(:pageview),
+        build(:pageview)
+      ])
+
+      populate_stats(site2, [
+        build(:pageview),
+        build(:event, name: "custom events")
+      ])
+
+      assert Billing.usage(user) == 4
     end
 
     test "only counts usage from sites where the user is the owner" do