plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-12-23 09:33:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Average Scroll Depth Metric: handle missing data better (#4889)

Browse Source 
* ingest missing pageleave as 255 for pageleave events

* return scroll depth as nil when no valid pageleave data in range

* also set empty comparison value as nil instead of 0

* add data migration
This commit is contained in:
RobertJoonas 2024-12-10 13:29:13 +01:00 committed by GitHub
parent c847d16a44
commit d21d48558a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 67 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/plausible/data_migration/missing_scroll_depth.ex Normal file
View File

@ -0,0 +1,18 @@
defmodule Plausible.DataMigration.MissingScrollDepth do
@moduledoc """
Set scroll_depth to 255 (max UInt8) for all pageleave events where it's 0.
"""
def run() do
Plausible.IngestRepo.query!(
"""
ALTER TABLE events_v2
#{Plausible.MigrationUtils.on_cluster_statement("events_v2")}
UPDATE scroll_depth = 255
WHERE name = 'pageleave' AND scroll_depth = 0
""",
[],
timeout: 60_000
)
end
end

2
lib/plausible/ingestion/request.ex
View File

@ -255,7 +255,7 @@ defmodule Plausible.Ingestion.Request do
case request_body["sd"] do case request_body["sd"] do
sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd
sd when is_integer(sd) and sd > 100 -> 100 sd when is_integer(sd) and sd > 100 -> 100
_ -> 0 _ -> 255
end end
Changeset.put_change(changeset, :scroll_depth, scroll_depth) Changeset.put_change(changeset, :scroll_depth, scroll_depth)

1
lib/plausible/stats/query_runner.ex
View File

@ -271,6 +271,7 @@ defmodule Plausible.Stats.QueryRunner do
do: nil do: nil
end end
defp empty_metric_value(:scroll_depth), do: nil
defp empty_metric_value(_), do: 0 defp empty_metric_value(_), do: 0
defp total_rows([]), do: 0 defp total_rows([]), do: 0

9
lib/plausible/stats/sql/special_metrics.ex
View File

@ -131,7 +131,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
if :scroll_depth in query.metrics do if :scroll_depth in query.metrics do
max_per_visitor_q = max_per_visitor_q =
Base.base_event_query(site, query) Base.base_event_query(site, query)
|> where([e], e.name == "pageleave") |> where([e], e.name == "pageleave" and e.scroll_depth <= 100)
|> select([e], %{ |> select([e], %{
user_id: e.user_id, user_id: e.user_id,
max_scroll_depth: max(e.scroll_depth) max_scroll_depth: max(e.scroll_depth)
@ -153,7 +153,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
scroll_depth_q = scroll_depth_q =
subquery(max_per_visitor_q) subquery(max_per_visitor_q)
|> select([p], %{ |> select([p], %{
scroll_depth: fragment("toUInt8(round(ifNotFinite(avg(?), 0)))", p.max_scroll_depth) scroll_depth:
fragment(
"if(isFinite(avg(?)), toUInt8(round(avg(?))), NULL)",
p.max_scroll_depth,
p.max_scroll_depth
)
}) })
|> select_merge(^dim_select) |> select_merge(^dim_select)
|> group_by(^dim_group_by) |> group_by(^dim_group_by)

15
test/plausible_web/controllers/api/external_controller_test.exs
View File

@ -1260,12 +1260,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site} {:ok, site: site}
end end
test "ingests scroll_depth as 0 when sd not in params", %{conn: conn, site: site} do test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain}) post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain}) post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "custom", u: "https://test.com", d: site.domain})
assert [%{scroll_depth: 0}, %{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site) pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
assert pageleave.scroll_depth == 255
end end
test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do
@ -1293,22 +1294,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageleave.scroll_depth == 100 assert pageleave.scroll_depth == 100
end end
test "ingests scroll_depth as 0 when sd is a string", %{conn: conn, site: site} do test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain}) post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"}) post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave")) pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
assert pageleave.scroll_depth == 0 assert pageleave.scroll_depth == 255
end end
test "ingests scroll_depth as 0 when sd is a negative integer", %{conn: conn, site: site} do test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain}) post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1}) post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave")) pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
assert pageleave.scroll_depth == 0 assert pageleave.scroll_depth == 255
end end
end end

4
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

@ -3773,7 +3773,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
}) })
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{"metrics" => [1, 0], "dimensions" => []} %{"metrics" => [1, nil], "dimensions" => []}
] ]
end end
@ -3787,7 +3787,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
}) })
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{"metrics" => [0], "dimensions" => []} %{"metrics" => [nil], "dimensions" => []}
] ]
end end

60
test/plausible_web/controllers/api/stats_controller/pages_test.exs
View File

@ -278,7 +278,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 600, "time_on_page" => 600,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/john-1", "name" => "/blog/john-1",
@ -286,7 +286,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -342,7 +342,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 120.0, "time_on_page" => 120.0,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/other-post", "name" => "/blog/other-post",
@ -350,7 +350,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 50, "bounce_rate" => 50,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/other-post", "name" => "/blog/other-post",
@ -404,7 +404,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -454,7 +454,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 100, "bounce_rate" => 100,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/john-1", "name" => "/blog/john-1",
@ -462,7 +462,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -598,7 +598,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3, "pageviews" => 3,
"bounce_rate" => 50, "bounce_rate" => 50,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -691,7 +691,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3, "pageviews" => 3,
"bounce_rate" => 50, "bounce_rate" => 50,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/about", "name" => "/about",
@ -699,7 +699,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 100, "bounce_rate" => 100,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -747,7 +747,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3, "pageviews" => 3,
"bounce_rate" => 50, "bounce_rate" => 50,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -794,7 +794,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 100, "bounce_rate" => 100,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/post-1", "name" => "/blog/post-1",
@ -802,7 +802,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/post-2", "name" => "/blog/post-2",
@ -810,7 +810,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -849,7 +849,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/blog/(/post-2", "name" => "/blog/(/post-2",
@ -857,7 +857,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"bounce_rate" => 50, "bounce_rate" => 50,
"time_on_page" => 600, "time_on_page" => 600,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"name" => "/about", "name" => "/about",
@ -912,7 +912,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => 0, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -1007,7 +1007,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 2, "visitors" => 2,
"pageviews" => 2, "pageviews" => 2,
"name" => "/", "name" => "/",
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"bounce_rate" => 0, "bounce_rate" => 0,
@ -1015,7 +1015,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"name" => "/some-other-page", "name" => "/some-other-page",
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -1056,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2, "pageviews" => 2,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 2, "visitors" => 2,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -1136,7 +1136,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3, "pageviews" => 3,
"time_on_page" => 1140.0, "time_on_page" => 1140.0,
"visitors" => 2, "visitors" => 2,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"bounce_rate" => 0, "bounce_rate" => 0,
@ -1144,7 +1144,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 1, "visitors" => 1,
"scroll_depth" => 0 "scroll_depth" => nil
} }
] ]
end end
@ -1492,20 +1492,20 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 0, "pageviews" => 0,
"time_on_page" => 0, "time_on_page" => 0,
"visitors" => 0, "visitors" => 0,
"scroll_depth" => 0, "scroll_depth" => nil,
"change" => %{ "change" => %{
"bounce_rate" => nil, "bounce_rate" => nil,
"pageviews" => 100, "pageviews" => 100,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 100, "visitors" => 100,
"scroll_depth" => 0 "scroll_depth" => nil
} }
}, },
"name" => "/page2", "name" => "/page2",
"pageviews" => 2, "pageviews" => 2,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 2, "visitors" => 2,
"scroll_depth" => 0 "scroll_depth" => nil
}, },
%{ %{
"bounce_rate" => 100, "bounce_rate" => 100,
@ -1513,19 +1513,19 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1, "pageviews" => 1,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 1, "visitors" => 1,
"scroll_depth" => 0, "scroll_depth" => nil,
"comparison" => %{ "comparison" => %{
"bounce_rate" => 100, "bounce_rate" => 100,
"pageviews" => 1, "pageviews" => 1,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 1, "visitors" => 1,
"scroll_depth" => 0, "scroll_depth" => nil,
"change" => %{ "change" => %{
"bounce_rate" => 0, "bounce_rate" => 0,
"pageviews" => 0, "pageviews" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 0, "visitors" => 0,
"scroll_depth" => 0 "scroll_depth" => nil
} }
} }
} }