Add support for imported custom events (#4033)

* Add Ecto schema for imported custom events

* Start importing custom events from GA4

* query imported goals

* make it possible to query events metric from imported

* make it possible to query pageviews in goal breakdown

* make it possible to query conversion rate

* fix rate limiting test

* add CR tests for dashboard API

* implement imported link_url breakdown

* override special custom event names coming from GA4

* allow specific goal filters in imported_q

* update GA4 import tests to use Stats API

* Improve tests slightly

* Update CHANGELOG.md

---------

Co-authored-by: Robert Joonas <robertjoonas16@gmail.com>

CHANGELOG.md

@@ -40,6 +40,8 @@ All notable changes to this project will be documented in this file.
 - Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906
 - Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
 - Add Yesterday as an time range option in the dashboard
+- Add support for importing Google Analytics 4 data
+- Import custom events from Google Analytics 4
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible/google/ga4/http.ex

@@ -31,6 +31,7 @@ defmodule Plausible.Google.GA4.HTTP do
               desc: true
             }
           ],
+          dimensionFilter: report_request.dimension_filter,
           limit: report_request.limit,
           offset: report_request.offset
         }

lib/plausible/google/ga4/report_request.ex

@@ -3,6 +3,13 @@ defmodule Plausible.Google.GA4.ReportRequest do
   Report request struct for Google Analytics 4 API
   """
 
+  @excluded_event_names [
+    "page_view",
+    "session_start",
+    "first_visit",
+    "user_engagement"
+  ]
+
   defstruct [
     :dataset,
     :dimensions,
@@ -11,6 +18,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
     :property,
     :access_token,
     :offset,
+    :dimension_filter,
     :limit
   ]
 
@@ -91,6 +99,25 @@ defmodule Plausible.Google.GA4.ReportRequest do
       #     "bounces = sessions - engagedSessions"
       #   ]
       # },
+      %__MODULE__{
+        dataset: "imported_custom_events",
+        dimensions: ["date", "eventName", "linkUrl"],
+        metrics: [
+          "totalUsers",
+          "eventCount"
+        ],
+        dimension_filter: %{
+          "notExpression" => %{
+            "filter" => %{
+              "fieldName" => "eventName",
+              "inListFilter" => %{
+                "values" => @excluded_event_names,
+                "caseSensitive" => true
+              }
+            }
+          }
+        }
+      },
       %__MODULE__{
         dataset: "imported_locations",
         dimensions: ["date", "countryId", "region", "city"],

lib/plausible/imported.ex

@@ -27,6 +27,7 @@ defmodule Plausible.Imported do
     Imported.Page,
     Imported.EntryPage,
     Imported.ExitPage,
+    Imported.CustomEvent,
     Imported.Location,
     Imported.Device,
     Imported.Browser,

lib/plausible/imported/buffer.ex

@@ -107,6 +107,7 @@ defmodule Plausible.Imported.Buffer do
   defp table_schema("imported_pages"), do: Plausible.Imported.Page
   defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage
   defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage
+  defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEvent
   defp table_schema("imported_locations"), do: Plausible.Imported.Location
   defp table_schema("imported_devices"), do: Plausible.Imported.Device
   defp table_schema("imported_browsers"), do: Plausible.Imported.Browser

lib/plausible/imported/custom_event.ex

@@ -0,0 +1,15 @@
+defmodule Plausible.Imported.CustomEvent do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_custom_events" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :name, :string
+    field :link_url, :string
+    field :visitors, Ch, type: "UInt64"
+    field :events, Ch, type: "UInt64"
+  end
+end

lib/plausible/imported/google_analytics4.ex

@@ -149,6 +149,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
     round(float)
   end
 
+  defp maybe_override_event_name("file_download"), do: "File Download"
+  defp maybe_override_event_name("click"), do: "Outbound Link: Click"
+  defp maybe_override_event_name(name), do: name
+
   defp new_from_report(site_id, import_id, "imported_visitors", row) do
     %{
       site_id: site_id,
@@ -229,6 +233,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
   #   }
   # end
 
+  defp new_from_report(site_id, import_id, "imported_custom_events", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(),
+      link_url: row.dimensions |> Map.fetch!("linkUrl"),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      events: row.metrics |> Map.fetch!("eventCount") |> parse_number()
+    }
+  end
+
   defp new_from_report(site_id, import_id, "imported_locations", row) do
     country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
     city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")

lib/plausible/stats/aggregate.ex

@@ -45,7 +45,7 @@ defmodule Plausible.Stats.Aggregate do
   defp aggregate_events(site, query, metrics) do
     from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
     |> merge_imported(site, query, metrics)
-    |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
+    |> maybe_add_conversion_rate(site, query, metrics)
     |> ClickhouseRepo.one()
   end
 

lib/plausible/stats/base.ex

@@ -577,7 +577,7 @@ defmodule Plausible.Stats.Base do
     |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
   end
 
-  defp total_visitors_subquery(site, query, true) do
+  defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
     dynamic(
       [e],
       selected_as(
@@ -588,16 +588,14 @@ defmodule Plausible.Stats.Base do
     )
   end
 
-  defp total_visitors_subquery(site, query, false) do
+  defp total_visitors_subquery(site, query) do
     dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
   end
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
       q
-      |> select_merge(
+      |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
-        ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}
-      )
       |> select_merge(%{
         percentage:
           fragment(
@@ -615,17 +613,13 @@ defmodule Plausible.Stats.Base do
   # Adds conversion_rate metric to query, calculated as
   # X / Y where Y is the same breakdown value without goal or props
   # filters.
-  def maybe_add_conversion_rate(q, site, query, metrics, opts) do
+  def maybe_add_conversion_rate(q, site, query, metrics) do
     if :conversion_rate in metrics do
-      include_imported = Keyword.fetch!(opts, :include_imported)
-
       total_query = query |> Query.remove_event_filters([:goal, :props])
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)
-      |> select_merge(
+      |> select_merge(^%{total_visitors: total_visitors_subquery(site, total_query)})
-        ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}
-      )
       |> select_merge([e], %{
         conversion_rate:
           fragment(

lib/plausible/stats/breakdown.ex

@@ -94,6 +94,7 @@ defmodule Plausible.Stats.Breakdown do
           }
         )
         |> select_merge(^select_columns)
+        |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select)
         |> apply_pagination(pagination)
       else
         nil
@@ -120,7 +121,7 @@ defmodule Plausible.Stats.Breakdown do
 
     if full_q do
       full_q
-      |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
+      |> maybe_add_conversion_rate(site, query, metrics)
       |> ClickhouseRepo.all()
       |> transform_keys(%{name: :goal})
       |> cast_revenue_metrics_to_money(revenue_goals)
@@ -149,7 +150,7 @@ defmodule Plausible.Stats.Breakdown do
     if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
 
     breakdown_events(site, query, metrics_to_select)
-    |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
+    |> maybe_add_conversion_rate(site, query, metrics)
     |> paginate_and_execute(metrics, pagination)
     |> transform_keys(%{breakdown_prop_value: custom_prop})
     |> Enum.map(&cast_revenue_metrics_to_money(&1, currency))

lib/plausible/stats/imported.ex

@@ -1,6 +1,6 @@
 defmodule Plausible.Stats.Imported do
   use Plausible.ClickhouseRepo
-  alias Plausible.Stats.Query
+  alias Plausible.Stats.{Query, Base}
 
   import Ecto.Query
   import Plausible.Stats.Fragments
@@ -26,11 +26,34 @@ defmodule Plausible.Stats.Imported do
     "visit:browser_version" => "imported_browsers",
     "visit:os" => "imported_operating_systems",
     "visit:os_version" => "imported_operating_systems",
-    "event:page" => "imported_pages"
+    "event:page" => "imported_pages",
+    "event:name" => "imported_custom_events",
+    "event:props:url" => "imported_custom_events"
   }
 
   @imported_properties Map.keys(@property_to_table_mappings)
 
+  def schema_supports_query?(query) do
+    filter_count = length(Map.keys(query.filters))
+
+    case {filter_count, query.property} do
+      {0, "event:props:" <> _} -> false
+      {0, _} -> true
+      {1, _} -> supports_single_filter?(query)
+      {_, _} -> false
+    end
+  end
+
+  defp supports_single_filter?(%Query{
+         filters: %{"event:goal" => {:is, {:event, event}}},
+         property: "event:props:url"
+       })
+       when event in ["Outbound Link: Click", "File Download"] do
+    true
+  end
+
+  defp supports_single_filter?(_query), do: false
+
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 
@@ -79,7 +102,6 @@ defmodule Plausible.Stats.Imported do
   end
 
   def merge_imported(q, _, %Query{include_imported: false}, _), do: q
-  def merge_imported(q, _, _, [:events | _]), do: q
 
   def merge_imported(q, site, %Query{property: property} = query, metrics)
       when property in @imported_properties do
@@ -96,12 +118,15 @@ defmodule Plausible.Stats.Imported do
         where: i.visitors > 0,
         select: %{}
       )
-      |> maybe_filter_by_breakdown_property(query.filters[property], dim)
+      |> maybe_apply_filter(query.filters, property, dim)
       |> group_imported_by(dim)
       |> select_imported_metrics(metrics)
 
     join_on =
       case dim do
+        :url ->
+          dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
+
         :os_version ->
           dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
 
@@ -137,6 +162,41 @@ defmodule Plausible.Stats.Imported do
 
   def merge_imported(q, _, _, _), do: q
 
+  def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
+
+  def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
+    page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
+
+    imported_q =
+      from(
+        i in "imported_pages",
+        where: i.site_id == ^site.id,
+        where: i.import_id in ^site.complete_import_ids,
+        where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
+        where: i.visitors > 0,
+        where:
+          fragment(
+            "notEmpty(multiMatchAllIndices(?, ?) as indices)",
+            i.page,
+            ^page_regexes
+          ),
+        array_join: index in fragment("indices"),
+        group_by: index,
+        select: %{
+          name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
+        }
+      )
+      |> select_imported_metrics(metrics)
+
+    from(s in Ecto.Query.subquery(q),
+      full_join: i in subquery(imported_q),
+      on: s.name == i.name,
+      select: %{}
+    )
+    |> select_joined_dimension(:name)
+    |> select_joined_metrics(metrics)
+  end
+
   def total_imported_visitors(site, query) do
     imported_visitors(site, query)
     |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
@@ -154,11 +214,22 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do
+  defp maybe_apply_filter(
-    where(q, [i], field(i, ^dim) in ^list)
+         q,
+         %{"event:goal" => {:is, {:event, event_name}}},
+         "event:props:url",
+         _dim
+       )
+       when event_name in ["Outbound Link: Click", "File Download"] do
+    where(q, [i], i.name == ^event_name)
   end
 
-  defp maybe_filter_by_breakdown_property(q, _, _), do: q
+  defp maybe_apply_filter(q, filters, property, dim) do
+    case filters[property] do
+      {:member, list} -> where(q, [i], field(i, ^dim) in ^list)
+      _ -> q
+    end
+  end
 
   defp select_imported_metrics(q, []), do: q
 
@@ -168,6 +239,21 @@ defmodule Plausible.Stats.Imported do
     |> select_imported_metrics(rest)
   end
 
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+         [:events | rest]
+       ) do
+    q
+    |> select_merge([i], %{events: sum(i.events)})
+    |> select_imported_metrics(rest)
+  end
+
+  defp select_imported_metrics(q, [:events | rest]) do
+    q
+    |> select_merge([i], %{events: sum(i.pageviews)})
+    |> select_imported_metrics(rest)
+  end
+
   defp select_imported_metrics(
          %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
          [:visits | rest]
@@ -192,6 +278,15 @@ defmodule Plausible.Stats.Imported do
     |> select_imported_metrics(rest)
   end
 
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+         [:pageviews | rest]
+       ) do
+    q
+    |> select_merge([i], %{pageviews: 0})
+    |> select_imported_metrics(rest)
+  end
+
   defp select_imported_metrics(q, [:pageviews | rest]) do
     q
     |> where([i], i.pageviews > 0)
@@ -351,6 +446,18 @@ defmodule Plausible.Stats.Imported do
     |> select_merge([i], %{^dim => field(i, ^dim)})
   end
 
+  defp group_imported_by(q, :name) do
+    q
+    |> group_by([i], i.name)
+    |> select_merge([i], %{name: i.name})
+  end
+
+  defp group_imported_by(q, :url) do
+    q
+    |> group_by([i], i.link_url)
+    |> select_merge([i], %{breakdown_prop_value: i.link_url})
+  end
+
   defp select_joined_dimension(q, :city) do
     select_merge(q, [s, i], %{
       city: fragment("greatest(?,?)", i.city, s.city)
@@ -372,6 +479,18 @@ defmodule Plausible.Stats.Imported do
     })
   end
 
+  defp select_joined_dimension(q, :url) do
+    select_merge(q, [s, i], %{
+      breakdown_prop_value:
+        fragment(
+          "if(empty(?), ?, ?)",
+          s.breakdown_prop_value,
+          i.breakdown_prop_value,
+          s.breakdown_prop_value
+        )
+    })
+  end
+
   defp select_joined_dimension(q, dim) do
     select_merge(q, [s, i], %{
       ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
@@ -397,6 +516,12 @@ defmodule Plausible.Stats.Imported do
     |> select_joined_metrics(rest)
   end
 
+  defp select_joined_metrics(q, [:events | rest]) do
+    q
+    |> select_merge([s, i], %{events: s.events + i.events})
+    |> select_joined_metrics(rest)
+  end
+
   defp select_joined_metrics(q, [:pageviews | rest]) do
     q
     |> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})

lib/plausible/stats/query.ex

@@ -14,7 +14,7 @@ defmodule Plausible.Stats.Query do
             experimental_reduced_joins?: false
 
   require OpenTelemetry.Tracer, as: Tracer
-  alias Plausible.Stats.{Filters, Interval}
+  alias Plausible.Stats.{Filters, Interval, Imported}
 
   @type t :: %__MODULE__{}
 
@@ -273,7 +273,7 @@ defmodule Plausible.Stats.Query do
     cond do
       is_nil(site.latest_import_end_date) -> false
       Date.after?(query.date_range.first, site.latest_import_end_date) -> false
-      Enum.any?(query.filters) -> false
+      not Imported.schema_supports_query?(query) -> false
       query.period == "realtime" -> false
       true -> requested?
     end

lib/plausible_web/controllers/api/external_stats_controller.ex

@@ -192,15 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
     end
   end
 
-  defp validate_metric("events" = metric, query) do
+  defp validate_metric(metric, _) when metric in ["visitors", "pageviews", "events"] do
-    if query.include_imported do
-      {:error, "Metric `#{metric}` cannot be queried with imported data"}
-    else
-      {:ok, metric}
-    end
-  end
-
-  defp validate_metric(metric, _) when metric in ["visitors", "pageviews"] do
     {:ok, metric}
   end
 

lib/plausible_web/controllers/api/stats_controller.ex

@@ -1204,9 +1204,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
     params = Map.put(params, "property", prefixed_prop)
 
-    query =
+    query = Query.from(site, params)
-      Query.from(site, params)
-      |> Map.put(:include_imported, false)
 
     metrics =
       if query.filters["event:goal"] do

lib/workers/clickhouse_clean_sites.ex

@@ -26,6 +26,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
     "imported_locations",
     "imported_operating_systems",
     "imported_pages",
+    "imported_custom_events",
     "imported_sources",
     "imported_visitors"
   ]

test/plausible/imported/google_analytics4_test.exs

@@ -16,6 +16,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
                       "fixture/ga4_report_imported_sources.json",
                       "fixture/ga4_report_imported_pages.json",
                       "fixture/ga4_report_imported_entry_pages.json",
+                      "fixture/ga4_report_imported_custom_events.json",
                       "fixture/ga4_report_imported_locations.json",
                       "fixture/ga4_report_imported_devices.json",
                       "fixture/ga4_report_imported_browsers.json",
@@ -89,6 +90,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
             "imported_pages" -> 3340
             "imported_entry_pages" -> 2934
             "imported_exit_pages" -> 0
+            "imported_custom_events" -> 56
             "imported_locations" -> 2291
             "imported_devices" -> 93
             "imported_browsers" -> 233
@@ -122,9 +124,19 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
 
       conn = put_req_header(conn, "authorization", "Bearer #{api_key}")
 
-      assert_timeseries(conn, common_params)
+      insert(:goal, event_name: "Outbound Link: Click", site: site)
-      assert_pages(conn, common_params)
+      insert(:goal, event_name: "view_search_results", site: site)
+      insert(:goal, event_name: "scroll", site: site)
 
+      # Timeseries
+      assert_timeseries(conn, common_params)
+
+      # Breakdown (event:*)
+      assert_pages(conn, common_params)
+      assert_custom_events(conn, common_params)
+      assert_outbound_link_urls(conn, common_params)
+
+      # Breakdown (visit:*)
       assert_sources(conn, breakdown_params)
       assert_utm_mediums(conn, breakdown_params)
       assert_entry_pages(conn, breakdown_params)
@@ -133,6 +145,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
       assert_browsers(conn, breakdown_params)
       assert_os(conn, breakdown_params)
       assert_os_versions(conn, breakdown_params)
+
+      # Misc
       assert_active_visitors(site_import)
     end
 
@@ -202,6 +216,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
             "imported_devices" -> 0
             "imported_browsers" -> 0
             "imported_operating_systems" -> 0
+            "imported_custom_events" -> 0
           end
 
         query = from(imported in table, where: imported.site_id == ^site.id)
@@ -229,6 +244,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
             "imported_devices" -> 93
             "imported_browsers" -> 233
             "imported_operating_systems" -> 1068
+            "imported_custom_events" -> 56
           end
 
         query = from(imported in table, where: imported.site_id == ^site.id)
@@ -265,6 +281,67 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
     end)
   end
 
+  defp assert_custom_events(conn, params) do
+    params =
+      params
+      |> Map.put("metrics", "visitors,events,conversion_rate")
+      |> Map.put("property", "event:goal")
+
+    %{"results" => results} =
+      get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
+
+    assert results == [
+             %{
+               "goal" => "scroll",
+               "visitors" => 1513,
+               "events" => 2130,
+               "conversion_rate" => 24.7
+             },
+             %{
+               "goal" => "Outbound Link: Click",
+               "visitors" => 17,
+               "events" => 17,
+               "conversion_rate" => 0.3
+             },
+             %{
+               "goal" => "view_search_results",
+               "visitors" => 11,
+               "events" => 30,
+               "conversion_rate" => 0.2
+             }
+           ]
+  end
+
+  defp assert_outbound_link_urls(conn, params) do
+    params =
+      Map.merge(params, %{
+        "metrics" => "visitors,events,conversion_rate",
+        "property" => "event:props:url",
+        "filters" => "event:goal==Outbound Link: Click"
+      })
+
+    %{"results" => results} =
+      get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
+
+    assert length(results) == 10
+
+    assert List.first(results) ==
+             %{
+               "url" => "https://www.facebook.com/kuhinjskeprice",
+               "visitors" => 6,
+               "conversion_rate" => 0.1,
+               "events" => 6
+             }
+
+    assert %{
+             "url" =>
+               "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/",
+             "visitors" => 1,
+             "conversion_rate" => 0.0,
+             "events" => 1
+           } in results
+  end
+
   defp assert_timeseries(conn, params) do
     params =
       Map.put(

test/plausible/purge_test.exs

@@ -30,6 +30,7 @@ defmodule Plausible.PurgeTest do
       build(:imported_pages),
       build(:imported_entry_pages),
       build(:imported_exit_pages),
+      build(:imported_custom_events),
       build(:imported_locations),
       build(:imported_devices),
       build(:imported_browsers),
@@ -43,6 +44,7 @@ defmodule Plausible.PurgeTest do
       build(:imported_pages),
       build(:imported_entry_pages),
       build(:imported_exit_pages),
+      build(:imported_custom_events),
       build(:imported_locations),
       build(:imported_devices),
       build(:imported_browsers),
@@ -101,6 +103,7 @@ defmodule Plausible.PurgeTest do
       build(:imported_pages),
       build(:imported_entry_pages),
       build(:imported_exit_pages),
+      build(:imported_custom_events),
       build(:imported_locations),
       build(:imported_devices),
       build(:imported_browsers),

test/plausible/stats/query_test.exs

@@ -204,4 +204,79 @@ defmodule Plausible.Stats.QueryTest do
       assert q.filters["visit:source"] == {:is, "Twitter"}
     end
   end
+
+  describe "include_imported" do
+    setup [:create_site]
+
+    test "is true when requested via params and imported data exists", %{site: site} do
+      insert(:site_import, site: site)
+      site = Plausible.Imported.load_import_data(site)
+
+      assert %{include_imported: true} =
+               Query.from(site, %{"period" => "day", "with_imported" => "true"})
+    end
+
+    test "is false when imported data does not exist", %{site: site} do
+      assert %{include_imported: false} =
+               Query.from(site, %{"period" => "day", "with_imported" => "true"})
+    end
+
+    test "is false when imported data exists but is out of the date range", %{site: site} do
+      insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])
+      site = Plausible.Imported.load_import_data(site)
+
+      assert %{include_imported: false} =
+               Query.from(site, %{"period" => "day", "with_imported" => "true"})
+    end
+
+    test "is false in realtime even when imported data from today exists", %{site: site} do
+      insert(:site_import, site: site)
+      site = Plausible.Imported.load_import_data(site)
+
+      assert %{include_imported: false} =
+               Query.from(site, %{"period" => "realtime", "with_imported" => "true"})
+    end
+
+    test "is false when an arbitrary custom property filter is used", %{site: site} do
+      insert(:site_import, site: site)
+      site = Plausible.Imported.load_import_data(site)
+
+      assert %{include_imported: false} =
+               Query.from(site, %{
+                 "period" => "day",
+                 "with_imported" => "true",
+                 "property" => "event:props:url",
+                 "filters" => Jason.encode!(%{"props" => %{"author" => "!John Doe"}})
+               })
+    end
+
+    test "is true when breaking down by url and filtering by outbound link or file download goal",
+         %{site: site} do
+      insert(:site_import, site: site)
+      site = Plausible.Imported.load_import_data(site)
+
+      Enum.each(["Outbound Link: Click", "File Download"], fn goal_name ->
+        assert %{include_imported: true} =
+                 Query.from(site, %{
+                   "period" => "day",
+                   "with_imported" => "true",
+                   "property" => "event:props:url",
+                   "filters" => Jason.encode!(%{"goal" => goal_name})
+                 })
+      end)
+    end
+
+    test "is false when breaking down by url but without a special goal filter",
+         %{site: site} do
+      insert(:site_import, site: site)
+      site = Plausible.Imported.load_import_data(site)
+
+      assert %{include_imported: false} =
+               Query.from(site, %{
+                 "period" => "day",
+                 "with_imported" => "true",
+                 "property" => "event:props:url"
+               })
+    end
+  end
 end

test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs

@@ -588,28 +588,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
                "visitors" => %{"value" => 1, "change" => 100}
              }
     end
-
-    test "events metric with imported data is disallowed", %{
-      conn: conn,
-      site: site,
-      site_import: site_import
-    } do
-      populate_stats(site, site_import.id, [
-        build(:imported_visitors, date: ~D[2023-01-01])
-      ])
-
-      conn =
-        get(conn, "/api/v1/stats/aggregate", %{
-          "site_id" => site.domain,
-          "period" => "day",
-          "date" => "2023-01-02",
-          "metrics" => "events",
-          "with_imported" => "true"
-        })
-
-      assert %{"error" => msg} = json_response(conn, 400)
-      assert msg == "Metric `events` cannot be queried with imported data"
-    end
   end
 
   describe "filters" do

test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs

@@ -3053,4 +3053,173 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
              }
     end
   end
+
+  describe "imported data" do
+    test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, page_path: "/test")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/breakdown", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "date" => "2021-01-01",
+          "property" => "event:goal",
+          "metrics" => "visitors,events,pageviews,conversion_rate",
+          "with_imported" => "true"
+        })
+
+      assert [
+               %{
+                 "goal" => "Purchase",
+                 "visitors" => 5,
+                 "events" => 7,
+                 "pageviews" => 0,
+                 "conversion_rate" => 62.5
+               },
+               %{
+                 "goal" => "Visit /test",
+                 "visitors" => 3,
+                 "events" => 3,
+                 "pageviews" => 3,
+                 "conversion_rate" => 37.5
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
+    test "pageviews are returned as events for breakdown reports other than custom events", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_operating_systems,
+          operating_system: "Mac",
+          pageviews: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])
+      ])
+
+      params = %{
+        "site_id" => site.domain,
+        "period" => "day",
+        "date" => "2021-01-01",
+        "metrics" => "events",
+        "with_imported" => "true"
+      }
+
+      breakdown_and_first = fn property ->
+        conn
+        |> get("/api/v1/stats/breakdown", Map.put(params, "property", property))
+        |> json_response(200)
+        |> Map.get("results")
+        |> List.first()
+      end
+
+      assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")
+      assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")
+      assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")
+      assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")
+      assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")
+      assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")
+      assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")
+      assert %{"source" => "Google", "events" => 1} = breakdown_and_first.("visit:source")
+    end
+
+    for goal_name <- ["Outbound Link: Click", "File Download"] do
+      test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+        insert(:goal, event_name: unquote(goal_name), site: site)
+        site_import = insert(:site_import, site: site)
+
+        populate_stats(site, site_import.id, [
+          build(:event,
+            name: unquote(goal_name),
+            "meta.key": ["url"],
+            "meta.value": ["https://one.com"]
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 2,
+            events: 5,
+            link_url: "https://one.com"
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 5,
+            events: 10,
+            link_url: "https://two.com"
+          ),
+          build(:imported_custom_events,
+            name: "some goal",
+            visitors: 5,
+            events: 10
+          ),
+          build(:imported_visitors, visitors: 9)
+        ])
+
+        conn =
+          get(conn, "/api/v1/stats/breakdown", %{
+            "site_id" => site.domain,
+            "period" => "day",
+            "property" => "event:props:url",
+            "filters" => "event:goal==#{unquote(goal_name)}",
+            "metrics" => "visitors,events,conversion_rate",
+            "with_imported" => "true"
+          })
+
+        assert json_response(conn, 200)["results"] == [
+                 %{
+                   "visitors" => 5,
+                   "url" => "https://two.com",
+                   "events" => 10,
+                   "conversion_rate" => 50.0
+                 },
+                 %{
+                   "visitors" => 3,
+                   "url" => "https://one.com",
+                   "events" => 6,
+                   "conversion_rate" => 30.0
+                 }
+               ]
+      end
+    end
+  end
 end

test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs

@@ -1525,4 +1525,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
              }
     end
   end
+
+  describe "imported data" do
+    test "returns pageviews as the value of events metric", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:imported_visitors, pageviews: 1, date: ~D[2021-01-01])
+      ])
+
+      first_result =
+        conn
+        |> get("/api/v1/stats/timeseries", %{
+          "site_id" => site.domain,
+          "period" => "7d",
+          "metrics" => "events",
+          "date" => "2021-01-07",
+          "with_imported" => "true"
+        })
+        |> json_response(200)
+        |> Map.get("results")
+        |> List.first()
+
+      assert first_result == %{"date" => "2021-01-01", "events" => 1}
+    end
+  end
 end

test/plausible_web/controllers/api/stats_controller/conversions_test.exs

@@ -626,44 +626,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
                }
              ]
     end
-
-    test "conversion_rate for goals should not be calculated with imported data", %{
-      conn: conn,
-      site: site
-    } do
-      site_import =
-        insert(:site_import,
-          start_date: ~D[2005-01-01],
-          end_date: Timex.today(),
-          source: :universal_analytics
-        )
-
-      populate_stats(site, site_import.id, [
-        build(:pageview, pathname: "/"),
-        build(:pageview, pathname: "/another"),
-        build(:pageview, pathname: "/blog/post-1"),
-        build(:pageview, pathname: "/blog/post-2"),
-        build(:imported_pages, page: "/blog/post-1"),
-        build(:imported_visitors)
-      ])
-
-      insert(:goal, %{site: site, page_path: "/blog**"})
-
-      conn =
-        get(
-          conn,
-          "/api/stats/#{site.domain}/conversions?period=day"
-        )
-
-      assert json_response(conn, 200) == [
-               %{
-                 "name" => "Visit /blog**",
-                 "visitors" => 2,
-                 "events" => 2,
-                 "conversion_rate" => 50
-               }
-             ]
-    end
   end
 
   describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do
@@ -785,4 +747,99 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
              ]
     end
   end
+
+  describe "GET /api/stats/:domain/conversions - with imported data" do
+    setup [:create_user, :log_in, :create_site]
+
+    test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, page_path: "/test")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      url_query_params = "?period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+      assert [
+               %{
+                 "name" => "Purchase",
+                 "visitors" => 5,
+                 "events" => 7,
+                 "conversion_rate" => 62.5
+               },
+               %{
+                 "name" => "Visit /test",
+                 "visitors" => 3,
+                 "events" => 3,
+                 "conversion_rate" => 37.5
+               }
+             ] = json_response(conn, 200)
+    end
+
+    test "calculates conversion_rate for goals with glob pattern with imported data", %{
+      conn: conn,
+      site: site
+    } do
+      site_import =
+        insert(:site_import,
+          start_date: ~D[2005-01-01],
+          end_date: Timex.today(),
+          source: :universal_analytics
+        )
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, pathname: "/"),
+        build(:pageview, pathname: "/another"),
+        build(:pageview, pathname: "/blog/post-1"),
+        build(:pageview, pathname: "/blog/post-2"),
+        build(:imported_pages, page: "/blog/post-1"),
+        build(:imported_visitors)
+      ])
+
+      insert(:goal, %{site: site, page_path: "/blog**"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/conversions?period=day"
+        )
+
+      assert json_response(conn, 200) == [
+               %{
+                 "name" => "Visit /blog**",
+                 "visitors" => 2,
+                 "events" => 2,
+                 "conversion_rate" => 50
+               }
+             ]
+    end
+  end
 end

test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs

@@ -1132,4 +1132,64 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
              }
     end
   end
+
+  describe "with imported data" do
+    setup [:create_user, :log_in, :create_new_site]
+
+    for goal_name <- ["Outbound Link: Click", "File Download"] do
+      test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+        insert(:goal, event_name: unquote(goal_name), site: site)
+        site_import = insert(:site_import, site: site)
+
+        populate_stats(site, site_import.id, [
+          build(:event,
+            name: unquote(goal_name),
+            "meta.key": ["url"],
+            "meta.value": ["https://one.com"]
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 2,
+            events: 5,
+            link_url: "https://one.com"
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 5,
+            events: 10,
+            link_url: "https://two.com"
+          ),
+          build(:imported_custom_events,
+            name: "view_search_results",
+            visitors: 100,
+            events: 200
+          ),
+          build(:imported_visitors, visitors: 9)
+        ])
+
+        filters = Jason.encode!(%{goal: unquote(goal_name)})
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{
+                   "visitors" => 5,
+                   "name" => "https://two.com",
+                   "events" => 10,
+                   "conversion_rate" => 50.0
+                 },
+                 %{
+                   "visitors" => 3,
+                   "name" => "https://one.com",
+                   "events" => 6,
+                   "conversion_rate" => 30.0
+                 }
+               ]
+      end
+    end
+  end
 end

test/support/factory.ex

@@ -237,6 +237,17 @@ defmodule Plausible.Factory do
     }
   end
 
+  def imported_custom_events_factory do
+    %{
+      table: "imported_custom_events",
+      date: Timex.today(),
+      name: "",
+      link_url: "",
+      visitors: 1,
+      events: 1
+    }
+  end
+
   def imported_locations_factory do
     %{
       table: "imported_locations",