From 29ae3a2c21c72076c05ba14149129d94549fd4ef Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 6 Jan 2020 15:51:43 +0200 Subject: [PATCH] Show bounce rate for referrers and pages --- assets/css/app.css | 4 + assets/js/dashboard/stats/modals/pages.js | 39 +++-- .../stats/modals/referrer-drilldown.js | 40 +++-- assets/js/dashboard/stats/modals/referrers.js | 52 ++++-- lib/plausible/ingest/session.ex | 1 + lib/plausible/session/schema.ex | 3 +- lib/plausible/stats/stats.ex | 150 +++++++++++++++++- .../controllers/api/stats_controller.ex | 9 +- ...00107095234_add_entry_page_to_sessions.exs | 25 +++ .../api/stats_controller/pages_test.exs | 16 ++ .../api/stats_controller/referrers_test.exs | 35 ++++ test/support/factory.ex | 1 + 12 files changed, 325 insertions(+), 50 deletions(-) create mode 100644 priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs diff --git a/assets/css/app.css b/assets/css/app.css index 5467a00e4..a14e269b8 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -218,3 +218,7 @@ a { width: 1em; transform: translateY(0.15em); } + +.table-striped tbody tr:nth-child(odd) { + background-color: #f1f5f8; +} diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js index 0029a935a..503f98d63 100644 --- a/assets/js/dashboard/stats/modals/pages.js +++ b/assets/js/dashboard/stats/modals/pages.js @@ -4,7 +4,6 @@ import { withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' import numberFormatter from '../../number-formatter' -import Bar from '../bar' import {parseQuery} from '../../query' class PagesModal extends React.Component { @@ -16,19 +15,25 @@ class PagesModal extends React.Component { componentDidMount() { const query = parseQuery(this.props.location.search, this.props.site) - api.get(`/api/stats/${this.props.site.domain}/pages`, query, {limit: 100}) + api.get(`/api/stats/${this.props.site.domain}/pages`, query, {limit: 100, include: 'bounce_rate'}) .then((res) => this.setState({loading: false, pages: res})) } + formatBounceRate(page) { + if (page.bounce_rate) { + return page.bounce_rate + '%' + } else { + return '-' + } + } + renderPage(page) { return ( - -
- { page.name } - {numberFormatter(page.count)} -
- - + + {page.name} + {numberFormatter(page.count)} + {this.formatBounceRate(page)} + ) } @@ -43,13 +48,21 @@ class PagesModal extends React.Component {

Top pages

-
by pageviews
-
- { this.state.pages.map(this.renderPage.bind(this)) } -
+ + + + + + + + + + { this.state.pages.map(this.renderPage.bind(this)) } + +
Page urlPageviewsBounce rate
) diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index ad38fce6e..2d4e5a959 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -4,7 +4,6 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' import numberFormatter from '../../number-formatter' -import Bar from '../bar' import {parseQuery, toHuman} from '../../query' class ReferrerDrilldownModal extends React.Component { @@ -17,19 +16,27 @@ class ReferrerDrilldownModal extends React.Component { } componentDidMount() { - api.get(`/api/stats/${this.props.site.domain}/referrers/${this.props.match.params.referrer}`, this.state.query, {limit: 100}) + api.get(`/api/stats/${this.props.site.domain}/referrers/${this.props.match.params.referrer}`, this.state.query, {limit: 100, include: 'bounce_rate'}) .then((res) => this.setState({loading: false, referrers: res.referrers, totalVisitors: res.total_visitors})) } + formatBounceRate(ref) { + if (ref.bounce_rate) { + return ref.bounce_rate + '%' + } else { + return '-' + } + } + renderReferrer(referrer) { return ( - -
- { referrer.name } - {numberFormatter(referrer.count)} -
- - + + + { referrer.name } + + {numberFormatter(referrer.count)} + {this.formatBounceRate(referrer)} + ) } @@ -58,9 +65,18 @@ class ReferrerDrilldownModal extends React.Component {

{this.state.totalVisitors} visitors from {this.props.match.params.referrer}
{toHuman(this.state.query)}

{this.renderGoalText()} -
- { this.state.referrers.map(this.renderReferrer.bind(this)) } -
+ + + + + + + + + + { this.state.referrers.map(this.renderReferrer.bind(this)) } + +
ReferrerVisitorsBounce rate
) diff --git a/assets/js/dashboard/stats/modals/referrers.js b/assets/js/dashboard/stats/modals/referrers.js index e00b551d1..206f0a4a1 100644 --- a/assets/js/dashboard/stats/modals/referrers.js +++ b/assets/js/dashboard/stats/modals/referrers.js @@ -4,31 +4,45 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' import numberFormatter from '../../number-formatter' -import Bar from '../bar' import {parseQuery} from '../../query' class ReferrersModal extends React.Component { constructor(props) { super(props) - this.state = {loading: true} + this.state = { + loading: true, + query: parseQuery(props.location.search, props.site) + } } componentDidMount() { - const query = parseQuery(this.props.location.search, this.props.site) + const include = this.showBounceRate() ? 'bounce_rate' : null - api.get(`/api/stats/${this.props.site.domain}/referrers`, query, {limit: 100}) + api.get(`/api/stats/${this.props.site.domain}/referrers`, this.state.query, {limit: 100, include: include}) .then((res) => this.setState({loading: false, referrers: res})) } + showBounceRate() { + return !this.state.query.filters.goal + } + + formatBounceRate(page) { + if (page.bounce_rate) { + return page.bounce_rate + '%' + } else { + return '-' + } + } + renderReferrer(referrer) { return ( - -
+ + { referrer.name } - {numberFormatter(referrer.count)} -
- - + + {numberFormatter(referrer.count)} + {this.showBounceRate() && {this.formatBounceRate(referrer)} } + ) } @@ -41,15 +55,23 @@ class ReferrersModal extends React.Component { return (
-

Referrers

+

Top Referrers

-
by visitors
-
- { this.state.referrers.map(this.renderReferrer.bind(this)) } -
+ + + + + + {this.showBounceRate() && } + + + + { this.state.referrers.map(this.renderReferrer.bind(this)) } + +
ReferrerVisitorsBounce rate
 ) diff --git a/lib/plausible/ingest/session.ex b/lib/plausible/ingest/session.ex index ec982d504..6b46c63cf 100644 --- a/lib/plausible/ingest/session.ex +++ b/lib/plausible/ingest/session.ex @@ -49,6 +49,7 @@ defmodule Plausible.Ingest.Session do hostname: event.hostname, user_id: event.user_id, new_visitor: event.new_visitor, + entry_page: event.pathname, is_bounce: state[:is_bounce], length: length, referrer: event.referrer, diff --git a/lib/plausible/session/schema.ex b/lib/plausible/session/schema.ex index 33a0a5d7b..7f1f54102 100644 --- a/lib/plausible/session/schema.ex +++ b/lib/plausible/session/schema.ex @@ -10,6 +10,7 @@ defmodule Plausible.Session do field :start, :naive_datetime, null: false field :length, :integer field :is_bounce, :boolean + field :entry_page, :string field :referrer, :string field :referrer_source, :string @@ -23,7 +24,7 @@ defmodule Plausible.Session do def changeset(session, attrs) do session - |> cast(attrs, [:hostname, :referrer, :new_visitor, :user_id, :start, :length, :is_bounce, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) + |> cast(attrs, [:hostname, :entry_page, :referrer, :new_visitor, :user_id, :start, :length, :is_bounce, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) |> validate_required([:hostname, :new_visitor, :user_id, :is_bounce, :start]) end end diff --git a/lib/plausible/stats/stats.ex b/lib/plausible/stats/stats.ex index 897e7d3a7..66cc0b684 100644 --- a/lib/plausible/stats/stats.ex +++ b/lib/plausible/stats/stats.ex @@ -163,14 +163,60 @@ defmodule Plausible.Stats do )) end - def top_referrers(site, query, limit \\ 5) do - Repo.all(from e in base_query(site, query), + def top_referrers(site, query, limit \\ 5, include \\ []) do + referrers = Repo.all(from e in base_query(site, query), select: %{name: e.referrer_source, count: count(e.user_id, :distinct)}, group_by: e.referrer_source, where: not is_nil(e.referrer_source), order_by: [desc: 2], limit: ^limit ) + + if "bounce_rate" in include do + bounce_rates = bounce_rates_by_referrer_source(site, query, Enum.map(referrers, fn ref -> ref[:name] end)) + + Enum.map(referrers, fn referrer -> + Map.put(referrer, :bounce_rate, bounce_rates[referrer[:name]]) + end) + else + referrers + end + end + + defp bounce_rates_by_referrer_source(site, query, referrers) do + {first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone) + + total_sessions_by_referrer = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.referrer_source in ^referrers, + group_by: s.referrer_source, + select: {s.referrer_source, count(s.id)} + ) |> Enum.into(%{}) + + bounced_sessions_by_referrer = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.is_bounce, + where: s.referrer_source in ^referrers, + group_by: s.referrer_source, + select: {s.referrer_source, count(s.id)} + ) |> Enum.into(%{}) + + Enum.reduce(referrers, %{}, fn referrer, acc -> + total_sessions = Map.get(total_sessions_by_referrer, referrer, 0) + bounced_sessions = Map.get(bounced_sessions_by_referrer, referrer, 0) + + bounce_rate = if total_sessions > 0 do + round(bounced_sessions / total_sessions * 100) + end + + Map.put(acc, referrer, bounce_rate) + end) end def visitors_from_referrer(site, query, referrer) do @@ -181,23 +227,115 @@ defmodule Plausible.Stats do ) end - def referrer_drilldown(site, query, referrer) do - Repo.all(from e in base_query(site, query), + def referrer_drilldown(site, query, referrer, include \\ []) do + referring_urls = Repo.all(from e in base_query(site, query), select: %{name: e.referrer, count: count(e.user_id, :distinct)}, group_by: e.referrer, where: e.referrer_source == ^referrer, order_by: [desc: 2], limit: 100 ) + + if "bounce_rate" in include do + bounce_rates = bounce_rates_by_referring_url(site, query, Enum.map(referring_urls, fn ref -> ref[:name] end)) + + Enum.map(referring_urls, fn url -> + Map.put(url, :bounce_rate, bounce_rates[url[:name]]) + end) + else + referring_urls + end end - def top_pages(site, query, limit \\ 5) do - Repo.all(from e in base_query(site, query), + defp bounce_rates_by_referring_url(site, query, referring_urls) do + {first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone) + + total_sessions_by_url = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.referrer in ^referring_urls, + group_by: s.referrer, + select: {s.referrer, count(s.id)} + ) |> Enum.into(%{}) + + bounced_sessions_by_url = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.is_bounce, + where: s.referrer in ^referring_urls, + group_by: s.referrer, + select: {s.referrer, count(s.id)} + ) |> Enum.into(%{}) + + Enum.reduce(referring_urls, %{}, fn url, acc -> + total_sessions = Map.get(total_sessions_by_url, url, 0) + bounced_sessions = Map.get(bounced_sessions_by_url, url, 0) + + bounce_rate = if total_sessions > 0 do + round(bounced_sessions / total_sessions * 100) + end + + Map.put(acc, url, bounce_rate) + end) + end + + def top_pages(site, query, limit \\ 5, include \\ []) do + pages = Repo.all(from e in base_query(site, query), select: %{name: e.pathname, count: count(e.pathname)}, group_by: e.pathname, order_by: [desc: count(e.pathname)], limit: ^limit ) + + if "bounce_rate" in include do + bounce_rates = bounce_rates_by_page_url(site, query, Enum.map(pages, fn page -> page[:name] end)) + + Enum.map(pages, fn url -> + Map.put(url, :bounce_rate, bounce_rates[url[:name]]) + end) + else + pages + end + end + + defp bounce_rates_by_page_url(site, query, page_urls) do + {first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone) + + total_sessions_by_url = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.entry_page in ^page_urls, + group_by: s.entry_page, + select: {s.entry_page, count(s.id)} + ) |> Enum.into(%{}) + + bounced_sessions_by_url = Repo.all( + from s in Plausible.Session, + where: s.hostname == ^site.domain, + where: s.new_visitor, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.is_bounce, + where: s.entry_page in ^page_urls, + group_by: s.entry_page, + select: {s.entry_page, count(s.id)} + ) |> Enum.into(%{}) + + Enum.reduce(page_urls, %{}, fn url, acc -> + total_sessions = Map.get(total_sessions_by_url, url, 0) + bounced_sessions = Map.get(bounced_sessions_by_url, url, 0) + + bounce_rate = if total_sessions > 0 do + round(bounced_sessions / total_sessions * 100) + end + + Map.put(acc, url, bounce_rate) + end) end @available_screen_sizes ["Desktop", "Laptop", "Tablet", "Mobile"] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index edd9b29d3..823723341 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -68,8 +68,9 @@ defmodule PlausibleWeb.Api.StatsController do def referrers(conn, params) do site = conn.assigns[:site] query = Stats.Query.from(site.timezone, params) + include = if params["include"], do: String.split(params["include"], ","), else: [] - json(conn, Stats.top_referrers(site, query, params["limit"] || 5)) + json(conn, Stats.top_referrers(site, query, params["limit"] || 5, include)) end @@ -101,8 +102,9 @@ defmodule PlausibleWeb.Api.StatsController do def referrer_drilldown(conn, %{"referrer" => referrer} = params) do site = conn.assigns[:site] query = Stats.Query.from(site.timezone, params) + include = if params["include"], do: String.split(params["include"], ","), else: [] - referrers = Stats.referrer_drilldown(site, query, referrer) + referrers = Stats.referrer_drilldown(site, query, referrer, include) total_visitors = Stats.visitors_from_referrer(site, query, referrer) json(conn, %{referrers: referrers, total_visitors: total_visitors}) end @@ -110,8 +112,9 @@ defmodule PlausibleWeb.Api.StatsController do def pages(conn, params) do site = conn.assigns[:site] query = Stats.Query.from(site.timezone, params) + include = if params["include"], do: String.split(params["include"], ","), else: [] - json(conn, Stats.top_pages(site, query, params["limit"] || 5)) + json(conn, Stats.top_pages(site, query, params["limit"] || 5, include)) end def countries(conn, params) do diff --git a/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs b/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs new file mode 100644 index 000000000..da0acecc9 --- /dev/null +++ b/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs @@ -0,0 +1,25 @@ +defmodule Plausible.Repo.Migrations.AddEntryPageToSessions do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :entry_page, :text + end + + execute """ + UPDATE sessions SET entry_page = pathname + FROM events + WHERE events.user_id = sessions.user_id + AND events.name = 'pageview' + AND events.new_visitor + """ + + execute """ + DELETE FROM sessions WHERE entry_page is null + """ + + alter table(:sessions) do + modify :entry_page, :text, null: false + end + end +end diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index b6d9d0a82..efe8b9bba 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -17,5 +17,21 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do %{"name" => "/contact", "count" => 1}, ] end + + test "calculates bounce rate for pages", %{conn: conn, site: site} do + insert(:pageview, hostname: site.domain, pathname: "/", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, pathname: "/", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, pathname: "/contact", timestamp: ~N[2019-01-01 02:00:00]) + + insert(:session, hostname: site.domain, entry_page: "/", is_bounce: true, start: ~N[2019-01-01 02:00:00]) + insert(:session, hostname: site.domain, entry_page: "/", is_bounce: false, start: ~N[2019-01-01 02:00:00]) + + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&include=bounce_rate") + + assert json_response(conn, 200) == [ + %{"name" => "/", "count" => 2, "bounce_rate" => 50}, + %{"name" => "/contact", "count" => 1, "bounce_rate" => nil}, + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index 38b657d93..7959978ef 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -19,6 +19,22 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ] end + test "calculates bounce rate for referrers", %{conn: conn, site: site} do + insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Bing", timestamp: ~N[2019-01-01 02:00:00]) + + insert(:session, hostname: site.domain, referrer_source: "Google", is_bounce: true, start: ~N[2019-01-01 02:00:00]) + insert(:session, hostname: site.domain, referrer_source: "Google", is_bounce: false, start: ~N[2019-01-01 02:00:00]) + + conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate") + + assert json_response(conn, 200) == [ + %{"name" => "Google", "count" => 2, "bounce_rate" => 50}, + %{"name" => "Bing", "count" => 1, "bounce_rate" => nil}, + ] + end + test "filters referrers for a custom goal", %{conn: conn, site: site} do insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) @@ -85,6 +101,25 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do } end + test "calculates bounce rate for referrer urls", %{conn: conn, site: site} do + insert(:pageview, hostname: site.domain, referrer_source: "10words", referrer: "10words.io/hello", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "10words", referrer: "10words.io/hello", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "10words", referrer: "10words.io/", timestamp: ~N[2019-01-01 02:00:00]) + + insert(:session, hostname: site.domain, referrer_source: "10words", referrer: "10words.io/hello", is_bounce: true, start: ~N[2019-01-01 02:00:00]) + insert(:session, hostname: site.domain, referrer_source: "10words", referrer: "10words.io/hello",is_bounce: false, start: ~N[2019-01-01 02:00:00]) + + conn = get(conn, "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&include=bounce_rate") + + assert json_response(conn, 200) == %{ + "total_visitors" => 3, + "referrers" => [ + %{"name" => "10words.io/hello", "count" => 2, "bounce_rate" => 50}, + %{"name" => "10words.io/", "count" => 1, "bounce_rate" => nil}, + ] + } + end + test "gets keywords from Google", %{conn: conn, user: user, site: site} do insert(:google_auth, user: user, user: user,site: site, property: "sc-domain:example.com") insert(:pageview, hostname: site.domain, referrer: "google.com", referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) diff --git a/test/support/factory.ex b/test/support/factory.ex index 2326e3b48..b266c7478 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -28,6 +28,7 @@ defmodule Plausible.Factory do %Plausible.Session{ hostname: hostname, new_visitor: true, + entry_page: "/", user_id: UUID.uuid4(), start: Timex.now(), is_bounce: false