From 4e7e932a75192ec3bb7f863027e0a8bb461c615f Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 9 May 2024 14:13:19 +0200 Subject: [PATCH] Add support for imported custom events (#4033) * Add Ecto schema for imported custom events * Start importing custom events from GA4 * query imported goals * make it possible to query events metric from imported * make it possible to query pageviews in goal breakdown * make it possible to query conversion rate * fix rate limiting test * add CR tests for dashboard API * implement imported link_url breakdown * override special custom event names coming from GA4 * allow specific goal filters in imported_q * update GA4 import tests to use Stats API * Improve tests slightly * Update CHANGELOG.md --------- Co-authored-by: Robert Joonas --- CHANGELOG.md | 2 + .../ga4_report_imported_custom_events.json | 1212 +++++++++++++++++ lib/plausible/google/ga4/http.ex | 1 + lib/plausible/google/ga4/report_request.ex | 27 + lib/plausible/imported.ex | 1 + lib/plausible/imported/buffer.ex | 1 + lib/plausible/imported/custom_event.ex | 15 + lib/plausible/imported/google_analytics4.ex | 16 + lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 16 +- lib/plausible/stats/breakdown.ex | 5 +- lib/plausible/stats/imported.ex | 139 +- lib/plausible/stats/query.ex | 4 +- .../api/external_stats_controller.ex | 10 +- .../controllers/api/stats_controller.ex | 4 +- lib/workers/clickhouse_clean_sites.ex | 1 + .../imported/google_analytics4_test.exs | 81 +- test/plausible/purge_test.exs | 3 + test/plausible/stats/query_test.exs | 75 + .../aggregate_test.exs | 22 - .../breakdown_test.exs | 169 +++ .../timeseries_test.exs | 28 + .../api/stats_controller/conversions_test.exs | 133 +- .../custom_prop_breakdown_test.exs | 60 + test/support/factory.ex | 11 + 25 files changed, 1941 insertions(+), 97 deletions(-) create mode 100644 fixture/ga4_report_imported_custom_events.json create mode 100644 lib/plausible/imported/custom_event.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 86048662a..1fe27d7b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ All notable changes to this project will be documented in this file. - Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906 - Add restrictive `robots.txt` for self-hosted plausible/analytics#3905 - Add Yesterday as an time range option in the dashboard +- Add support for importing Google Analytics 4 data +- Import custom events from Google Analytics 4 ### Removed - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions diff --git a/fixture/ga4_report_imported_custom_events.json b/fixture/ga4_report_imported_custom_events.json new file mode 100644 index 000000000..c11673676 --- /dev/null +++ b/fixture/ga4_report_imported_custom_events.json @@ -0,0 +1,1212 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "eventName" + }, + { + "name": "linkUrl" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Amsterdam" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "eventCount", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 56, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240131" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "40" + }, + { + "value": "55" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240130" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "42" + }, + { + "value": "61" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240129" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "40" + }, + { + "value": "57" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240128" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "67" + }, + { + "value": "85" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240128" + }, + { + "value": "click" + }, + { + "value": "https://www.facebook.com/kuhinjskeprice" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240128" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240127" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "51" + }, + { + "value": "65" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240127" + }, + { + "value": "click" + }, + { + "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240126" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "50" + }, + { + "value": "70" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240126" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240125" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "58" + }, + { + "value": "98" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240125" + }, + { + "value": "click" + }, + { + "value": "https://www.google.co.uk/imgres?imgurl=http://30.media.tumblr.com/oapJteChehjbp5traCrAENnXo1_500.jpg" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240125" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240124" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "61" + }, + { + "value": "97" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240124" + }, + { + "value": "click" + }, + { + "value": "http://www.ikea.com/dk/da/catalog/products/20213835/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240124" + }, + { + "value": "click" + }, + { + "value": "https://www.facebook.com/kuhinjskeprice" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240123" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "74" + }, + { + "value": "114" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240123" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240122" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "68" + }, + { + "value": "95" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240122" + }, + { + "value": "click" + }, + { + "value": "https://pinterest.com/pin/68539225549526297/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240122" + }, + { + "value": "click" + }, + { + "value": "https://www.facebook.com/kuhinjskeprice" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240121" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "116" + }, + { + "value": "181" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240121" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240120" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "111" + }, + { + "value": "190" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240120" + }, + { + "value": "click" + }, + { + "value": "https://www.facebook.com/kuhinjskeprice" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240120" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "6" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240119" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "119" + }, + { + "value": "186" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240119" + }, + { + "value": "click" + }, + { + "value": "https://www.facebook.com/kuhinjskeprice" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240119" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240118" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "86" + }, + { + "value": "125" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240117" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "43" + }, + { + "value": "55" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240116" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "66" + }, + { + "value": "95" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240116" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240115" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "29" + }, + { + "value": "32" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240114" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "31" + }, + { + "value": "36" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240113" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "30" + }, + { + "value": "34" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240113" + }, + { + "value": "click" + }, + { + "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240113" + }, + { + "value": "click" + }, + { + "value": "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240112" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "31" + }, + { + "value": "33" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240112" + }, + { + "value": "click" + }, + { + "value": "http://www.ikea.com/dk/da/catalog/products/80010222/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240111" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "26" + }, + { + "value": "29" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240111" + }, + { + "value": "click" + }, + { + "value": "http://www.apartmenttherapy.com/minh-and-ted-bought-a-distress-150686" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240110" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "24" + }, + { + "value": "28" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240109" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "32" + }, + { + "value": "56" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240108" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "24" + }, + { + "value": "26" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240108" + }, + { + "value": "click" + }, + { + "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240107" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "27" + }, + { + "value": "45" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240107" + }, + { + "value": "view_search_results" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "15" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240106" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "22" + }, + { + "value": "22" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240105" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "38" + }, + { + "value": "39" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240104" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "24" + }, + { + "value": "27" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240103" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "34" + }, + { + "value": "35" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240103" + }, + { + "value": "click" + }, + { + "value": "https://pinterest.com/pin/238198267760892524/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240102" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "29" + }, + { + "value": "39" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240102" + }, + { + "value": "click" + }, + { + "value": "http://www.lonelyplanet.com/denmark/images/legoland-copenhagen$25938-2" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240101" + }, + { + "value": "scroll" + }, + { + "value": "" + } + ], + "metricValues": [ + { + "value": "20" + }, + { + "value": "20" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex index 615b02bdc..39eb97d42 100644 --- a/lib/plausible/google/ga4/http.ex +++ b/lib/plausible/google/ga4/http.ex @@ -31,6 +31,7 @@ defmodule Plausible.Google.GA4.HTTP do desc: true } ], + dimensionFilter: report_request.dimension_filter, limit: report_request.limit, offset: report_request.offset } diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex index 5647ae19a..7a94d8b0f 100644 --- a/lib/plausible/google/ga4/report_request.ex +++ b/lib/plausible/google/ga4/report_request.ex @@ -3,6 +3,13 @@ defmodule Plausible.Google.GA4.ReportRequest do Report request struct for Google Analytics 4 API """ + @excluded_event_names [ + "page_view", + "session_start", + "first_visit", + "user_engagement" + ] + defstruct [ :dataset, :dimensions, @@ -11,6 +18,7 @@ defmodule Plausible.Google.GA4.ReportRequest do :property, :access_token, :offset, + :dimension_filter, :limit ] @@ -91,6 +99,25 @@ defmodule Plausible.Google.GA4.ReportRequest do # "bounces = sessions - engagedSessions" # ] # }, + %__MODULE__{ + dataset: "imported_custom_events", + dimensions: ["date", "eventName", "linkUrl"], + metrics: [ + "totalUsers", + "eventCount" + ], + dimension_filter: %{ + "notExpression" => %{ + "filter" => %{ + "fieldName" => "eventName", + "inListFilter" => %{ + "values" => @excluded_event_names, + "caseSensitive" => true + } + } + } + } + }, %__MODULE__{ dataset: "imported_locations", dimensions: ["date", "countryId", "region", "city"], diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex index 3a6f8543e..c914314a3 100644 --- a/lib/plausible/imported.ex +++ b/lib/plausible/imported.ex @@ -27,6 +27,7 @@ defmodule Plausible.Imported do Imported.Page, Imported.EntryPage, Imported.ExitPage, + Imported.CustomEvent, Imported.Location, Imported.Device, Imported.Browser, diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex index fccbb6278..a1c50131a 100644 --- a/lib/plausible/imported/buffer.ex +++ b/lib/plausible/imported/buffer.ex @@ -107,6 +107,7 @@ defmodule Plausible.Imported.Buffer do defp table_schema("imported_pages"), do: Plausible.Imported.Page defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage + defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEvent defp table_schema("imported_locations"), do: Plausible.Imported.Location defp table_schema("imported_devices"), do: Plausible.Imported.Device defp table_schema("imported_browsers"), do: Plausible.Imported.Browser diff --git a/lib/plausible/imported/custom_event.ex b/lib/plausible/imported/custom_event.ex new file mode 100644 index 000000000..c52b9fcfc --- /dev/null +++ b/lib/plausible/imported/custom_event.ex @@ -0,0 +1,15 @@ +defmodule Plausible.Imported.CustomEvent do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_custom_events" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :name, :string + field :link_url, :string + field :visitors, Ch, type: "UInt64" + field :events, Ch, type: "UInt64" + end +end diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex index 0b4670124..0b72c7a3f 100644 --- a/lib/plausible/imported/google_analytics4.ex +++ b/lib/plausible/imported/google_analytics4.ex @@ -149,6 +149,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do round(float) end + defp maybe_override_event_name("file_download"), do: "File Download" + defp maybe_override_event_name("click"), do: "Outbound Link: Click" + defp maybe_override_event_name(name), do: name + defp new_from_report(site_id, import_id, "imported_visitors", row) do %{ site_id: site_id, @@ -229,6 +233,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 do # } # end + defp new_from_report(site_id, import_id, "imported_custom_events", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(), + link_url: row.dimensions |> Map.fetch!("linkUrl"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + events: row.metrics |> Map.fetch!("eventCount") |> parse_number() + } + end + defp new_from_report(site_id, import_id, "imported_locations", row) do country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("") city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("") diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index e1055e420..991e80431 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -45,7 +45,7 @@ defmodule Plausible.Stats.Aggregate do defp aggregate_events(site, query, metrics) do from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) |> merge_imported(site, query, metrics) - |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported) + |> maybe_add_conversion_rate(site, query, metrics) |> ClickhouseRepo.one() end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 298437880..fe74fb426 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -577,7 +577,7 @@ defmodule Plausible.Stats.Base do |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) end - defp total_visitors_subquery(site, query, true) do + defp total_visitors_subquery(site, %Query{include_imported: true} = query) do dynamic( [e], selected_as( @@ -588,16 +588,14 @@ defmodule Plausible.Stats.Base do ) end - defp total_visitors_subquery(site, query, false) do + defp total_visitors_subquery(site, query) do dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) end def add_percentage_metric(q, site, query, metrics) do if :percentage in metrics do q - |> select_merge( - ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)} - ) + |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)}) |> select_merge(%{ percentage: fragment( @@ -615,17 +613,13 @@ defmodule Plausible.Stats.Base do # Adds conversion_rate metric to query, calculated as # X / Y where Y is the same breakdown value without goal or props # filters. - def maybe_add_conversion_rate(q, site, query, metrics, opts) do + def maybe_add_conversion_rate(q, site, query, metrics) do if :conversion_rate in metrics do - include_imported = Keyword.fetch!(opts, :include_imported) - total_query = query |> Query.remove_event_filters([:goal, :props]) # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL subquery(q) - |> select_merge( - ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)} - ) + |> select_merge(^%{total_visitors: total_visitors_subquery(site, total_query)}) |> select_merge([e], %{ conversion_rate: fragment( diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 7512df1a3..566da8a07 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -94,6 +94,7 @@ defmodule Plausible.Stats.Breakdown do } ) |> select_merge(^select_columns) + |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select) |> apply_pagination(pagination) else nil @@ -120,7 +121,7 @@ defmodule Plausible.Stats.Breakdown do if full_q do full_q - |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) + |> maybe_add_conversion_rate(site, query, metrics) |> ClickhouseRepo.all() |> transform_keys(%{name: :goal}) |> cast_revenue_metrics_to_money(revenue_goals) @@ -149,7 +150,7 @@ defmodule Plausible.Stats.Breakdown do if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) breakdown_events(site, query, metrics_to_select) - |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) + |> maybe_add_conversion_rate(site, query, metrics) |> paginate_and_execute(metrics, pagination) |> transform_keys(%{breakdown_prop_value: custom_prop}) |> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 5906a5858..9ab3dd703 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -1,6 +1,6 @@ defmodule Plausible.Stats.Imported do use Plausible.ClickhouseRepo - alias Plausible.Stats.Query + alias Plausible.Stats.{Query, Base} import Ecto.Query import Plausible.Stats.Fragments @@ -26,11 +26,34 @@ defmodule Plausible.Stats.Imported do "visit:browser_version" => "imported_browsers", "visit:os" => "imported_operating_systems", "visit:os_version" => "imported_operating_systems", - "event:page" => "imported_pages" + "event:page" => "imported_pages", + "event:name" => "imported_custom_events", + "event:props:url" => "imported_custom_events" } @imported_properties Map.keys(@property_to_table_mappings) + def schema_supports_query?(query) do + filter_count = length(Map.keys(query.filters)) + + case {filter_count, query.property} do + {0, "event:props:" <> _} -> false + {0, _} -> true + {1, _} -> supports_single_filter?(query) + {_, _} -> false + end + end + + defp supports_single_filter?(%Query{ + filters: %{"event:goal" => {:is, {:event, event}}}, + property: "event:props:url" + }) + when event in ["Outbound Link: Click", "File Download"] do + true + end + + defp supports_single_filter?(_query), do: false + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _), do: native_q @@ -79,7 +102,6 @@ defmodule Plausible.Stats.Imported do end def merge_imported(q, _, %Query{include_imported: false}, _), do: q - def merge_imported(q, _, _, [:events | _]), do: q def merge_imported(q, site, %Query{property: property} = query, metrics) when property in @imported_properties do @@ -96,12 +118,15 @@ defmodule Plausible.Stats.Imported do where: i.visitors > 0, select: %{} ) - |> maybe_filter_by_breakdown_property(query.filters[property], dim) + |> maybe_apply_filter(query.filters, property, dim) |> group_imported_by(dim) |> select_imported_metrics(metrics) join_on = case dim do + :url -> + dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value) + :os_version -> dynamic([s, i], s.os == i.os and s.os_version == i.os_version) @@ -137,6 +162,41 @@ defmodule Plausible.Stats.Imported do def merge_imported(q, _, _, _), do: q + def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q + + def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do + page_regexes = Enum.map(page_exprs, &Base.page_regex/1) + + imported_q = + from( + i in "imported_pages", + where: i.site_id == ^site.id, + where: i.import_id in ^site.complete_import_ids, + where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, + where: i.visitors > 0, + where: + fragment( + "notEmpty(multiMatchAllIndices(?, ?) as indices)", + i.page, + ^page_regexes + ), + array_join: index in fragment("indices"), + group_by: index, + select: %{ + name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) + } + ) + |> select_imported_metrics(metrics) + + from(s in Ecto.Query.subquery(q), + full_join: i in subquery(imported_q), + on: s.name == i.name, + select: %{} + ) + |> select_joined_dimension(:name) + |> select_joined_metrics(metrics) + end + def total_imported_visitors(site, query) do imported_visitors(site, query) |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) @@ -154,11 +214,22 @@ defmodule Plausible.Stats.Imported do ) end - defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do - where(q, [i], field(i, ^dim) in ^list) + defp maybe_apply_filter( + q, + %{"event:goal" => {:is, {:event, event_name}}}, + "event:props:url", + _dim + ) + when event_name in ["Outbound Link: Click", "File Download"] do + where(q, [i], i.name == ^event_name) end - defp maybe_filter_by_breakdown_property(q, _, _), do: q + defp maybe_apply_filter(q, filters, property, dim) do + case filters[property] do + {:member, list} -> where(q, [i], field(i, ^dim) in ^list) + _ -> q + end + end defp select_imported_metrics(q, []), do: q @@ -168,6 +239,21 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end + defp select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, + [:events | rest] + ) do + q + |> select_merge([i], %{events: sum(i.events)}) + |> select_imported_metrics(rest) + end + + defp select_imported_metrics(q, [:events | rest]) do + q + |> select_merge([i], %{events: sum(i.pageviews)}) + |> select_imported_metrics(rest) + end + defp select_imported_metrics( %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, [:visits | rest] @@ -192,6 +278,15 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end + defp select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, + [:pageviews | rest] + ) do + q + |> select_merge([i], %{pageviews: 0}) + |> select_imported_metrics(rest) + end + defp select_imported_metrics(q, [:pageviews | rest]) do q |> where([i], i.pageviews > 0) @@ -351,6 +446,18 @@ defmodule Plausible.Stats.Imported do |> select_merge([i], %{^dim => field(i, ^dim)}) end + defp group_imported_by(q, :name) do + q + |> group_by([i], i.name) + |> select_merge([i], %{name: i.name}) + end + + defp group_imported_by(q, :url) do + q + |> group_by([i], i.link_url) + |> select_merge([i], %{breakdown_prop_value: i.link_url}) + end + defp select_joined_dimension(q, :city) do select_merge(q, [s, i], %{ city: fragment("greatest(?,?)", i.city, s.city) @@ -372,6 +479,18 @@ defmodule Plausible.Stats.Imported do }) end + defp select_joined_dimension(q, :url) do + select_merge(q, [s, i], %{ + breakdown_prop_value: + fragment( + "if(empty(?), ?, ?)", + s.breakdown_prop_value, + i.breakdown_prop_value, + s.breakdown_prop_value + ) + }) + end + defp select_joined_dimension(q, dim) do select_merge(q, [s, i], %{ ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim)) @@ -397,6 +516,12 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(rest) end + defp select_joined_metrics(q, [:events | rest]) do + q + |> select_merge([s, i], %{events: s.events + i.events}) + |> select_joined_metrics(rest) + end + defp select_joined_metrics(q, [:pageviews | rest]) do q |> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews}) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 00bf937ed..5af5ede7f 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -14,7 +14,7 @@ defmodule Plausible.Stats.Query do experimental_reduced_joins?: false require OpenTelemetry.Tracer, as: Tracer - alias Plausible.Stats.{Filters, Interval} + alias Plausible.Stats.{Filters, Interval, Imported} @type t :: %__MODULE__{} @@ -273,7 +273,7 @@ defmodule Plausible.Stats.Query do cond do is_nil(site.latest_import_end_date) -> false Date.after?(query.date_range.first, site.latest_import_end_date) -> false - Enum.any?(query.filters) -> false + not Imported.schema_supports_query?(query) -> false query.period == "realtime" -> false true -> requested? end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 3d2df1cb3..062387399 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -192,15 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end end - defp validate_metric("events" = metric, query) do - if query.include_imported do - {:error, "Metric `#{metric}` cannot be queried with imported data"} - else - {:ok, metric} - end - end - - defp validate_metric(metric, _) when metric in ["visitors", "pageviews"] do + defp validate_metric(metric, _) when metric in ["visitors", "pageviews", "events"] do {:ok, metric} end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index a36b3f679..66659f3e5 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -1204,9 +1204,7 @@ defmodule PlausibleWeb.Api.StatsController do params = Map.put(params, "property", prefixed_prop) - query = - Query.from(site, params) - |> Map.put(:include_imported, false) + query = Query.from(site, params) metrics = if query.filters["event:goal"] do diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex index 0d2475d06..fbe67ee6f 100644 --- a/lib/workers/clickhouse_clean_sites.ex +++ b/lib/workers/clickhouse_clean_sites.ex @@ -26,6 +26,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do "imported_locations", "imported_operating_systems", "imported_pages", + "imported_custom_events", "imported_sources", "imported_visitors" ] diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs index 84ca05095..d89912c1d 100644 --- a/test/plausible/imported/google_analytics4_test.exs +++ b/test/plausible/imported/google_analytics4_test.exs @@ -16,6 +16,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do "fixture/ga4_report_imported_sources.json", "fixture/ga4_report_imported_pages.json", "fixture/ga4_report_imported_entry_pages.json", + "fixture/ga4_report_imported_custom_events.json", "fixture/ga4_report_imported_locations.json", "fixture/ga4_report_imported_devices.json", "fixture/ga4_report_imported_browsers.json", @@ -89,6 +90,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do "imported_pages" -> 3340 "imported_entry_pages" -> 2934 "imported_exit_pages" -> 0 + "imported_custom_events" -> 56 "imported_locations" -> 2291 "imported_devices" -> 93 "imported_browsers" -> 233 @@ -122,9 +124,19 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do conn = put_req_header(conn, "authorization", "Bearer #{api_key}") - assert_timeseries(conn, common_params) - assert_pages(conn, common_params) + insert(:goal, event_name: "Outbound Link: Click", site: site) + insert(:goal, event_name: "view_search_results", site: site) + insert(:goal, event_name: "scroll", site: site) + # Timeseries + assert_timeseries(conn, common_params) + + # Breakdown (event:*) + assert_pages(conn, common_params) + assert_custom_events(conn, common_params) + assert_outbound_link_urls(conn, common_params) + + # Breakdown (visit:*) assert_sources(conn, breakdown_params) assert_utm_mediums(conn, breakdown_params) assert_entry_pages(conn, breakdown_params) @@ -133,6 +145,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert_browsers(conn, breakdown_params) assert_os(conn, breakdown_params) assert_os_versions(conn, breakdown_params) + + # Misc assert_active_visitors(site_import) end @@ -202,6 +216,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do "imported_devices" -> 0 "imported_browsers" -> 0 "imported_operating_systems" -> 0 + "imported_custom_events" -> 0 end query = from(imported in table, where: imported.site_id == ^site.id) @@ -229,6 +244,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do "imported_devices" -> 93 "imported_browsers" -> 233 "imported_operating_systems" -> 1068 + "imported_custom_events" -> 56 end query = from(imported in table, where: imported.site_id == ^site.id) @@ -265,6 +281,67 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do end) end + defp assert_custom_events(conn, params) do + params = + params + |> Map.put("metrics", "visitors,events,conversion_rate") + |> Map.put("property", "event:goal") + + %{"results" => results} = + get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) + + assert results == [ + %{ + "goal" => "scroll", + "visitors" => 1513, + "events" => 2130, + "conversion_rate" => 24.7 + }, + %{ + "goal" => "Outbound Link: Click", + "visitors" => 17, + "events" => 17, + "conversion_rate" => 0.3 + }, + %{ + "goal" => "view_search_results", + "visitors" => 11, + "events" => 30, + "conversion_rate" => 0.2 + } + ] + end + + defp assert_outbound_link_urls(conn, params) do + params = + Map.merge(params, %{ + "metrics" => "visitors,events,conversion_rate", + "property" => "event:props:url", + "filters" => "event:goal==Outbound Link: Click" + }) + + %{"results" => results} = + get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) + + assert length(results) == 10 + + assert List.first(results) == + %{ + "url" => "https://www.facebook.com/kuhinjskeprice", + "visitors" => 6, + "conversion_rate" => 0.1, + "events" => 6 + } + + assert %{ + "url" => + "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/", + "visitors" => 1, + "conversion_rate" => 0.0, + "events" => 1 + } in results + end + defp assert_timeseries(conn, params) do params = Map.put( diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exs index a06ba9061..14cbbc783 100644 --- a/test/plausible/purge_test.exs +++ b/test/plausible/purge_test.exs @@ -30,6 +30,7 @@ defmodule Plausible.PurgeTest do build(:imported_pages), build(:imported_entry_pages), build(:imported_exit_pages), + build(:imported_custom_events), build(:imported_locations), build(:imported_devices), build(:imported_browsers), @@ -43,6 +44,7 @@ defmodule Plausible.PurgeTest do build(:imported_pages), build(:imported_entry_pages), build(:imported_exit_pages), + build(:imported_custom_events), build(:imported_locations), build(:imported_devices), build(:imported_browsers), @@ -101,6 +103,7 @@ defmodule Plausible.PurgeTest do build(:imported_pages), build(:imported_entry_pages), build(:imported_exit_pages), + build(:imported_custom_events), build(:imported_locations), build(:imported_devices), build(:imported_browsers), diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 3e38c748e..f56858954 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -204,4 +204,79 @@ defmodule Plausible.Stats.QueryTest do assert q.filters["visit:source"] == {:is, "Twitter"} end end + + describe "include_imported" do + setup [:create_site] + + test "is true when requested via params and imported data exists", %{site: site} do + insert(:site_import, site: site) + site = Plausible.Imported.load_import_data(site) + + assert %{include_imported: true} = + Query.from(site, %{"period" => "day", "with_imported" => "true"}) + end + + test "is false when imported data does not exist", %{site: site} do + assert %{include_imported: false} = + Query.from(site, %{"period" => "day", "with_imported" => "true"}) + end + + test "is false when imported data exists but is out of the date range", %{site: site} do + insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01]) + site = Plausible.Imported.load_import_data(site) + + assert %{include_imported: false} = + Query.from(site, %{"period" => "day", "with_imported" => "true"}) + end + + test "is false in realtime even when imported data from today exists", %{site: site} do + insert(:site_import, site: site) + site = Plausible.Imported.load_import_data(site) + + assert %{include_imported: false} = + Query.from(site, %{"period" => "realtime", "with_imported" => "true"}) + end + + test "is false when an arbitrary custom property filter is used", %{site: site} do + insert(:site_import, site: site) + site = Plausible.Imported.load_import_data(site) + + assert %{include_imported: false} = + Query.from(site, %{ + "period" => "day", + "with_imported" => "true", + "property" => "event:props:url", + "filters" => Jason.encode!(%{"props" => %{"author" => "!John Doe"}}) + }) + end + + test "is true when breaking down by url and filtering by outbound link or file download goal", + %{site: site} do + insert(:site_import, site: site) + site = Plausible.Imported.load_import_data(site) + + Enum.each(["Outbound Link: Click", "File Download"], fn goal_name -> + assert %{include_imported: true} = + Query.from(site, %{ + "period" => "day", + "with_imported" => "true", + "property" => "event:props:url", + "filters" => Jason.encode!(%{"goal" => goal_name}) + }) + end) + end + + test "is false when breaking down by url but without a special goal filter", + %{site: site} do + insert(:site_import, site: site) + site = Plausible.Imported.load_import_data(site) + + assert %{include_imported: false} = + Query.from(site, %{ + "period" => "day", + "with_imported" => "true", + "property" => "event:props:url" + }) + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 8561eb3d8..cb248f1f6 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -588,28 +588,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do "visitors" => %{"value" => 1, "change" => 100} } end - - test "events metric with imported data is disallowed", %{ - conn: conn, - site: site, - site_import: site_import - } do - populate_stats(site, site_import.id, [ - build(:imported_visitors, date: ~D[2023-01-01]) - ]) - - conn = - get(conn, "/api/v1/stats/aggregate", %{ - "site_id" => site.domain, - "period" => "day", - "date" => "2023-01-02", - "metrics" => "events", - "with_imported" => "true" - }) - - assert %{"error" => msg} = json_response(conn, 400) - assert msg == "Metric `events` cannot be queried with imported data" - end end describe "filters" do diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 178c7e932..0b1ae8504 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -3053,4 +3053,173 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do } end end + + describe "imported data" do + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Purchase") + insert(:goal, site: site, page_path: "/test") + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2021-01-01] + ), + build(:imported_pages, + page: "/test", + visitors: 2, + pageviews: 2, + date: ~D[2021-01-01] + ), + build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:goal", + "metrics" => "visitors,events,pageviews,conversion_rate", + "with_imported" => "true" + }) + + assert [ + %{ + "goal" => "Purchase", + "visitors" => 5, + "events" => 7, + "pageviews" => 0, + "conversion_rate" => 62.5 + }, + %{ + "goal" => "Visit /test", + "visitors" => 3, + "events" => 3, + "pageviews" => 3, + "conversion_rate" => 37.5 + } + ] = json_response(conn, 200)["results"] + end + + test "pageviews are returned as events for breakdown reports other than custom events", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_operating_systems, + operating_system: "Mac", + pageviews: 1, + date: ~D[2021-01-01] + ), + build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01]) + ]) + + params = %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "events", + "with_imported" => "true" + } + + breakdown_and_first = fn property -> + conn + |> get("/api/v1/stats/breakdown", Map.put(params, "property", property)) + |> json_response(200) + |> Map.get("results") + |> List.first() + end + + assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser") + assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device") + assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page") + assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page") + assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country") + assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os") + assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page") + assert %{"source" => "Google", "events" => 1} = breakdown_and_first.("visit:source") + end + + for goal_name <- ["Outbound Link: Click", "File Download"] do + test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["url"], + "meta.value": ["https://one.com"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + link_url: "https://one.com" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + link_url: "https://two.com" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "property" => "event:props:url", + "filters" => "event:goal==#{unquote(goal_name)}", + "metrics" => "visitors,events,conversion_rate", + "with_imported" => "true" + }) + + assert json_response(conn, 200)["results"] == [ + %{ + "visitors" => 5, + "url" => "https://two.com", + "events" => 10, + "conversion_rate" => 50.0 + }, + %{ + "visitors" => 3, + "url" => "https://one.com", + "events" => 6, + "conversion_rate" => 30.0 + } + ] + end + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs index f9d6186b4..75bc9655e 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs @@ -1525,4 +1525,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do } end end + + describe "imported data" do + test "returns pageviews as the value of events metric", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_visitors, pageviews: 1, date: ~D[2021-01-01]) + ]) + + first_result = + conn + |> get("/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "7d", + "metrics" => "events", + "date" => "2021-01-07", + "with_imported" => "true" + }) + |> json_response(200) + |> Map.get("results") + |> List.first() + + assert first_result == %{"date" => "2021-01-01", "events" => 1} + end + end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 5d8e9e9c5..f932456df 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -626,44 +626,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do } ] end - - test "conversion_rate for goals should not be calculated with imported data", %{ - conn: conn, - site: site - } do - site_import = - insert(:site_import, - start_date: ~D[2005-01-01], - end_date: Timex.today(), - source: :universal_analytics - ) - - populate_stats(site, site_import.id, [ - build(:pageview, pathname: "/"), - build(:pageview, pathname: "/another"), - build(:pageview, pathname: "/blog/post-1"), - build(:pageview, pathname: "/blog/post-2"), - build(:imported_pages, page: "/blog/post-1"), - build(:imported_visitors) - ]) - - insert(:goal, %{site: site, page_path: "/blog**"}) - - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "Visit /blog**", - "visitors" => 2, - "events" => 2, - "conversion_rate" => 50 - } - ] - end end describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do @@ -785,4 +747,99 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end end + + describe "GET /api/stats/:domain/conversions - with imported data" do + setup [:create_user, :log_in, :create_site] + + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Purchase") + insert(:goal, site: site, page_path: "/test") + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2021-01-01] + ), + build(:imported_pages, + page: "/test", + visitors: 2, + pageviews: 2, + date: ~D[2021-01-01] + ), + build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + ]) + + url_query_params = "?period=day&date=2021-01-01&with_imported=true" + conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}") + + assert [ + %{ + "name" => "Purchase", + "visitors" => 5, + "events" => 7, + "conversion_rate" => 62.5 + }, + %{ + "name" => "Visit /test", + "visitors" => 3, + "events" => 3, + "conversion_rate" => 37.5 + } + ] = json_response(conn, 200) + end + + test "calculates conversion_rate for goals with glob pattern with imported data", %{ + conn: conn, + site: site + } do + site_import = + insert(:site_import, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/another"), + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2"), + build(:imported_pages, page: "/blog/post-1"), + build(:imported_visitors) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/conversions?period=day" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Visit /blog**", + "visitors" => 2, + "events" => 2, + "conversion_rate" => 50 + } + ] + end + end end diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs index d53658806..9c1898a02 100644 --- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs @@ -1132,4 +1132,64 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do } end end + + describe "with imported data" do + setup [:create_user, :log_in, :create_new_site] + + for goal_name <- ["Outbound Link: Click", "File Download"] do + test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["url"], + "meta.value": ["https://one.com"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + link_url: "https://one.com" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + link_url: "https://two.com" + ), + build(:imported_custom_events, + name: "view_search_results", + visitors: 100, + events: 200 + ), + build(:imported_visitors, visitors: 9) + ]) + + filters = Jason.encode!(%{goal: unquote(goal_name)}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "visitors" => 5, + "name" => "https://two.com", + "events" => 10, + "conversion_rate" => 50.0 + }, + %{ + "visitors" => 3, + "name" => "https://one.com", + "events" => 6, + "conversion_rate" => 30.0 + } + ] + end + end + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 6ab32a750..d2856ed21 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -237,6 +237,17 @@ defmodule Plausible.Factory do } end + def imported_custom_events_factory do + %{ + table: "imported_custom_events", + date: Timex.today(), + name: "", + link_url: "", + visitors: 1, + events: 1 + } + end + def imported_locations_factory do %{ table: "imported_locations",