mirror of
https://github.com/plausible/analytics.git
synced 2024-12-22 17:11:36 +03:00
Average Scroll Depth Metric: handle missing data better (#4889)
* ingest missing pageleave as 255 for pageleave events * return scroll depth as nil when no valid pageleave data in range * also set empty comparison value as nil instead of 0 * add data migration
This commit is contained in:
RobertJoonas
GitHub
parent
c847d16a44
commit
d21d48558a
18
lib/plausible/data_migration/missing_scroll_depth.ex
Normal file
18
lib/plausible/data_migration/missing_scroll_depth.ex
Normal file
@ -0,0 +1,18 @@
|
||||
defmodule Plausible.DataMigration.MissingScrollDepth do
|
||||
@moduledoc """
|
||||
Set scroll_depth to 255 (max UInt8) for all pageleave events where it's 0.
|
||||
"""
|
||||
|
||||
def run() do
|
||||
Plausible.IngestRepo.query!(
|
||||
"""
|
||||
ALTER TABLE events_v2
|
||||
#{Plausible.MigrationUtils.on_cluster_statement("events_v2")}
|
||||
UPDATE scroll_depth = 255
|
||||
WHERE name = 'pageleave' AND scroll_depth = 0
|
||||
""",
|
||||
[],
|
||||
timeout: 60_000
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -255,7 +255,7 @@ defmodule Plausible.Ingestion.Request do
|
||||
case request_body["sd"] do
|
||||
sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd
|
||||
sd when is_integer(sd) and sd > 100 -> 100
|
||||
_ -> 0
|
||||
_ -> 255
|
||||
end
|
||||
|
||||
Changeset.put_change(changeset, :scroll_depth, scroll_depth)
|
||||
|
||||
@ -271,6 +271,7 @@ defmodule Plausible.Stats.QueryRunner do
|
||||
do: nil
|
||||
end
|
||||
|
||||
defp empty_metric_value(:scroll_depth), do: nil
|
||||
defp empty_metric_value(_), do: 0
|
||||
|
||||
defp total_rows([]), do: 0
|
||||
|
||||
@ -131,7 +131,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
|
||||
if :scroll_depth in query.metrics do
|
||||
max_per_visitor_q =
|
||||
Base.base_event_query(site, query)
|
||||
|> where([e], e.name == "pageleave")
|
||||
|> where([e], e.name == "pageleave" and e.scroll_depth <= 100)
|
||||
|> select([e], %{
|
||||
user_id: e.user_id,
|
||||
max_scroll_depth: max(e.scroll_depth)
|
||||
@ -153,7 +153,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
|
||||
scroll_depth_q =
|
||||
subquery(max_per_visitor_q)
|
||||
|> select([p], %{
|
||||
scroll_depth: fragment("toUInt8(round(ifNotFinite(avg(?), 0)))", p.max_scroll_depth)
|
||||
scroll_depth:
|
||||
fragment(
|
||||
"if(isFinite(avg(?)), toUInt8(round(avg(?))), NULL)",
|
||||
p.max_scroll_depth,
|
||||
p.max_scroll_depth
|
||||
)
|
||||
})
|
||||
|> select_merge(^dim_select)
|
||||
|> group_by(^dim_group_by)
|
||||
|
||||
@ -1260,12 +1260,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
{:ok, site: site}
|
||||
end
|
||||
|
||||
test "ingests scroll_depth as 0 when sd not in params", %{conn: conn, site: site} do
|
||||
test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} do
|
||||
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
|
||||
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})
|
||||
post(conn, "/api/event", %{n: "custom", u: "https://test.com", d: site.domain})
|
||||
|
||||
assert [%{scroll_depth: 0}, %{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)
|
||||
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
|
||||
|
||||
assert pageleave.scroll_depth == 255
|
||||
end
|
||||
|
||||
test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do
|
||||
@ -1293,22 +1294,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
assert pageleave.scroll_depth == 100
|
||||
end
|
||||
|
||||
test "ingests scroll_depth as 0 when sd is a string", %{conn: conn, site: site} do
|
||||
test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do
|
||||
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
|
||||
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})
|
||||
|
||||
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
|
||||
|
||||
assert pageleave.scroll_depth == 0
|
||||
assert pageleave.scroll_depth == 255
|
||||
end
|
||||
|
||||
test "ingests scroll_depth as 0 when sd is a negative integer", %{conn: conn, site: site} do
|
||||
test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do
|
||||
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
|
||||
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})
|
||||
|
||||
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
|
||||
|
||||
assert pageleave.scroll_depth == 0
|
||||
assert pageleave.scroll_depth == 255
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -3773,7 +3773,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
})
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{"metrics" => [1, 0], "dimensions" => []}
|
||||
%{"metrics" => [1, nil], "dimensions" => []}
|
||||
]
|
||||
end
|
||||
|
||||
@ -3787,7 +3787,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
})
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{"metrics" => [0], "dimensions" => []}
|
||||
%{"metrics" => [nil], "dimensions" => []}
|
||||
]
|
||||
end
|
||||
|
||||
|
||||
@ -278,7 +278,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 600,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/john-1",
|
||||
@ -286,7 +286,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -342,7 +342,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 120.0,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/other-post",
|
||||
@ -350,7 +350,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 50,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/other-post",
|
||||
@ -404,7 +404,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -454,7 +454,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 100,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/john-1",
|
||||
@ -462,7 +462,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -598,7 +598,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 3,
|
||||
"bounce_rate" => 50,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -691,7 +691,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 3,
|
||||
"bounce_rate" => 50,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/about",
|
||||
@ -699,7 +699,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 100,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -747,7 +747,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 3,
|
||||
"bounce_rate" => 50,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -794,7 +794,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 100,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/post-1",
|
||||
@ -802,7 +802,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/post-2",
|
||||
@ -810,7 +810,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -849,7 +849,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/blog/(/post-2",
|
||||
@ -857,7 +857,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"bounce_rate" => 50,
|
||||
"time_on_page" => 600,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"name" => "/about",
|
||||
@ -912,7 +912,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -1007,7 +1007,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"visitors" => 2,
|
||||
"pageviews" => 2,
|
||||
"name" => "/",
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => 0,
|
||||
@ -1015,7 +1015,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"name" => "/some-other-page",
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -1056,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 2,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 2,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -1136,7 +1136,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 3,
|
||||
"time_on_page" => 1140.0,
|
||||
"visitors" => 2,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => 0,
|
||||
@ -1144,7 +1144,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 1,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -1492,20 +1492,20 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 0,
|
||||
"time_on_page" => 0,
|
||||
"visitors" => 0,
|
||||
"scroll_depth" => 0,
|
||||
"scroll_depth" => nil,
|
||||
"change" => %{
|
||||
"bounce_rate" => nil,
|
||||
"pageviews" => 100,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 100,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
},
|
||||
"name" => "/page2",
|
||||
"pageviews" => 2,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 2,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => 100,
|
||||
@ -1513,19 +1513,19 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"pageviews" => 1,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 1,
|
||||
"scroll_depth" => 0,
|
||||
"scroll_depth" => nil,
|
||||
"comparison" => %{
|
||||
"bounce_rate" => 100,
|
||||
"pageviews" => 1,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 1,
|
||||
"scroll_depth" => 0,
|
||||
"scroll_depth" => nil,
|
||||
"change" => %{
|
||||
"bounce_rate" => 0,
|
||||
"pageviews" => 0,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 0,
|
||||
"scroll_depth" => 0
|
||||
"scroll_depth" => nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user