From 8d9667a949a2628af479f0f8205ae5731155e9c6 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 28 Sep 2020 11:29:24 +0300 Subject: [PATCH] Show utm_medium, utm_source, and utm_campaign in sources modal (#321) * Show utm_medium, utm_source, and utm_campaign in sources modal * Allow filtering by UTM tags * Integrate filters with URL bar * Add CHANGELOG entry * Refresh modal when filter changes * Remove Direct / None from campaign results * Add UTM tabs to top sources report * Add pagination * Remove dropdown from top sources popup * Fix bug in clickhouse_ecto * Remove referrers_for_goal * Make sure UTM tags work OK with goals * Make source tab selection sticky * Consistent styling for devices and source tabs * Add noref in realtime to utm tabs * Fix tests --- CHANGELOG.md | 3 + assets/js/dashboard/filters.js | 9 + assets/js/dashboard/query.js | 3 + assets/js/dashboard/router.js | 7 +- assets/js/dashboard/stats/devices.js | 24 +- .../stats/modals/{referrers.js => sources.js} | 82 ++++-- .../js/dashboard/stats/sources/source-list.js | 164 ++++++++++- lib/plausible/stats/clickhouse.ex | 270 +++++++++++++----- .../controllers/api/stats_controller.ex | 26 +- lib/plausible_web/router.ex | 6 +- lib/workers/send_email_report.ex | 2 +- mix.lock | 2 +- .../{referrers_test.exs => sources_test.exs} | 38 ++- test/support/clickhouse_setup.ex | 3 + 14 files changed, 488 insertions(+), 151 deletions(-) rename assets/js/dashboard/stats/modals/{referrers.js => sources.js} (56%) rename test/plausible_web/controllers/api/stats_controller/{referrers_test.exs => sources_test.exs} (78%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b26428b9c..56a992a1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,5 +3,8 @@ All notable changes to this project will be documented in this file. ## [1.0.0] - Unreleased +### Added +- Collect and present link tags (`utm_medium`, `utm_source`, `utm_campaign`) in the dashboard + ### Changed - Replace configuration parameters `CLICKHOUSE_DATABASE_{HOST,NAME,USER,PASSWORD}` with a single `CLICKHOUSE_DATABASE_URL` [plausible/analytics#317](https://github.com/plausible/analytics/pull/317) diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 8a70f71ba..42f8586f1 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -9,6 +9,15 @@ function filterText(key, value) { if (key === "source") { return Source: {value} } + if (key === "utm_medium") { + return UTM medium: {value} + } + if (key === "utm_source") { + return UTM source: {value} + } + if (key === "utm_campaign") { + return UTM campaign: {value} + } if (key === "referrer") { return Referrer: {value} } diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index 56ba7a80b..e2f9a434b 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -25,6 +25,9 @@ export function parseQuery(querystring, site) { filters: { 'goal': q.get('goal'), 'source': q.get('source'), + 'utm_medium': q.get('utm_medium'), + 'utm_source': q.get('utm_source'), + 'utm_campaign': q.get('utm_campaign'), 'referrer': q.get('referrer'), 'page': q.get('page') } diff --git a/assets/js/dashboard/router.js b/assets/js/dashboard/router.js index be328b894..18b171550 100644 --- a/assets/js/dashboard/router.js +++ b/assets/js/dashboard/router.js @@ -1,7 +1,6 @@ import React, { useEffect } from 'react'; import Dash from './index' -import Modal from './stats/modals/modal' -import ReferrersModal from './stats/modals/referrers' +import SourcesModal from './stats/modals/sources' import ReferrersDrilldownModal from './stats/modals/referrer-drilldown' import GoogleKeywordsModal from './stats/modals/google-keywords' import PagesModal from './stats/modals/pages' @@ -28,8 +27,8 @@ export default function Router({site, loggedIn}) { - - + + diff --git a/assets/js/dashboard/stats/devices.js b/assets/js/dashboard/stats/devices.js index 91a87d03e..23fe1357c 100644 --- a/assets/js/dashboard/stats/devices.js +++ b/assets/js/dashboard/stats/devices.js @@ -242,7 +242,11 @@ class OperatingSystems extends React.Component { export default class Devices extends React.Component { constructor(props) { super(props) - this.state = {mode: 'size'} + this.tabKey = 'deviceTab__' + props.site.domain + const storedTab = window.localStorage[this.tabKey] + this.state = { + mode: storedTab || 'size' + } } renderContent() { @@ -257,6 +261,7 @@ export default class Devices extends React.Component { setMode(mode) { return () => { + window.localStorage[this.tabKey] = mode this.setState({mode}) } } @@ -266,9 +271,9 @@ export default class Devices extends React.Component { const extraClass = name === 'OS' ? '' : ' border-r border-gray-300' if (isActive) { - return {name} + return
  • {name}
    • } else { - return {name} + return
  • {name}
    • } } @@ -276,12 +281,15 @@ export default class Devices extends React.Component { return (
    -

    Devices

    -
    - { this.renderPill('Size', 'size') } - { this.renderPill('Browser', 'browser') } - { this.renderPill('OS', 'os') } +
    +

    Devices

    + +
      + { this.renderPill('Size', 'size') } + { this.renderPill('Browser', 'browser') } + { this.renderPill('OS', 'os') } +
    { this.renderContent() } diff --git a/assets/js/dashboard/stats/modals/referrers.js b/assets/js/dashboard/stats/modals/sources.js similarity index 56% rename from assets/js/dashboard/stats/modals/referrers.js rename to assets/js/dashboard/stats/modals/sources.js index 97213ad45..068d63078 100644 --- a/assets/js/dashboard/stats/modals/referrers.js +++ b/assets/js/dashboard/stats/modals/sources.js @@ -1,40 +1,54 @@ import React from "react"; import { Link, withRouter } from 'react-router-dom' +import Transition from "../../../transition.js"; import FadeIn from '../../fade-in' import Modal from './modal' import * as api from '../../api' import numberFormatter, {durationFormatter} from '../../number-formatter' import {parseQuery} from '../../query' -class ReferrersModal extends React.Component { +const TITLES = { + sources: 'Top sources', + utm_mediums: 'Top UTM mediums', + utm_sources: 'Top UTM sources', + utm_campaigns: 'Top UTM campaigns' +} + +class SourcesModal extends React.Component { constructor(props) { super(props) this.state = { loading: true, - referrers: [], + sources: [], query: parseQuery(props.location.search, props.site), page: 1, moreResultsAvailable: false } } - loadReferrers() { - const {query, page, referrers} = this.state + loadSources() { + const {site} = this.props + const {query, page, sources} = this.state - if (query.filters.goal) { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers`, query, {limit: 100, page: page}) - .then((res) => this.setState({loading: false, referrers: referrers.concat(res), moreResultsAvailable: res.length === 100})) - } else { - const include = this.showExtra() ? 'bounce_rate,visit_duration' : null - - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers`, query, {limit: 100, page: page, include: include, show_noref: true}) - .then((res) => this.setState({loading: false, referrers: referrers.concat(res), moreResultsAvailable: res.length === 100})) - } + const include = this.showExtra() ? 'bounce_rate,visit_duration' : null + api.get(`/api/stats/${encodeURIComponent(site.domain)}/${this.currentFilter()}`, query, {limit: 100, page: page, include: include, show_noref: true}) + .then((res) => this.setState({loading: false, sources: sources.concat(res), moreResultsAvailable: res.length === 100})) } componentDidMount() { - this.loadReferrers() + this.loadSources() + } + + componentDidUpdate(prevProps) { + if (this.props.location.pathname !== prevProps.location.pathname) { + this.setState({sources: [], loading: true}, this.loadSources.bind(this)) + } + } + + currentFilter() { + const urlparts = this.props.location.pathname.split('/') + return urlparts[urlparts.length - 1] } showExtra() { @@ -42,7 +56,7 @@ class ReferrersModal extends React.Component { } loadMore() { - this.setState({loading: true, page: this.state.page + 1}, this.loadReferrers.bind(this)) + this.setState({loading: true, page: this.state.page + 1}, this.loadSources.bind(this)) } formatBounceRate(page) { @@ -53,27 +67,31 @@ class ReferrersModal extends React.Component { } } - formatDuration(referrer) { - if (typeof(referrer.visit_duration) === 'number') { - return durationFormatter(referrer.visit_duration) + formatDuration(source) { + if (typeof(source.visit_duration) === 'number') { + return durationFormatter(source.visit_duration) } else { return '-' } } - renderReferrer(referrer) { + renderSource(source) { const query = new URLSearchParams(window.location.search) - query.set('source', referrer.name) + const filter = this.currentFilter() + if (filter === 'sources') query.set('source', source.name) + if (filter === 'utm_mediums') query.set('utm_medium', source.name) + if (filter === 'utm_sources') query.set('utm_source', source.name) + if (filter === 'utm_campaigns') query.set('utm_campaign', source.name) return ( - + - - { referrer.name } + + { source.name } - {numberFormatter(referrer.count)} - {this.showExtra() && {this.formatBounceRate(referrer)} } - {this.showExtra() && {this.formatDuration(referrer)} } + {numberFormatter(source.count)} + {this.showExtra() && {this.formatBounceRate(source)} } + {this.showExtra() && {this.formatDuration(source)} } ) } @@ -96,10 +114,14 @@ class ReferrersModal extends React.Component { } } + title() { + return TITLES[this.currentFilter()] + } + render() { return ( -

    Top Sources

    +

    {this.title()}

    @@ -107,14 +129,14 @@ class ReferrersModal extends React.Component { - + {this.showExtra() && } {this.showExtra() && } - { this.state.referrers.map(this.renderReferrer.bind(this)) } + { this.state.sources.map(this.renderSource.bind(this)) }
    ReferrerSource {this.label()}Bounce rateVisit duration
    @@ -125,4 +147,4 @@ class ReferrersModal extends React.Component { } } -export default withRouter(ReferrersModal) +export default withRouter(SourcesModal) diff --git a/assets/js/dashboard/stats/sources/source-list.js b/assets/js/dashboard/stats/sources/source-list.js index eef7abdc3..faffbda46 100644 --- a/assets/js/dashboard/stats/sources/source-list.js +++ b/assets/js/dashboard/stats/sources/source-list.js @@ -8,7 +8,7 @@ import MoreLink from '../more-link' import numberFormatter from '../../number-formatter' import * as api from '../../api' -export default class Referrers extends React.Component { +class AllSources extends React.Component { constructor(props) { super(props) this.state = {loading: true} @@ -31,13 +31,8 @@ export default class Referrers extends React.Component { } fetchReferrers() { - if (this.props.query.filters.goal) { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers`, this.props.query) - .then((res) => this.setState({loading: false, referrers: res})) - } else { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers`, this.props.query, {show_noref: this.showNoRef()}) - .then((res) => this.setState({loading: false, referrers: res})) - } + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/sources`, this.props.query, {show_noref: this.showNoRef()}) + .then((res) => this.setState({loading: false, referrers: res})) } renderReferrer(referrer) { @@ -87,9 +82,12 @@ export default class Referrers extends React.Component { if (this.state.referrers) { return ( -

    Top sources

    +
    +

    Top sources

    + { this.props.renderTabs() } +
    { this.renderList() } - +     ) } @@ -106,3 +104,149 @@ export default class Referrers extends React.Component { ) } } + +const UTM_TAGS = { + utm_medium: {label: 'UTM Medium', endpoint: 'utm_mediums'}, + utm_source: {label: 'UTM Source', endpoint: 'utm_sources'}, + utm_campaign: {label: 'UTM Campaign', endpoint: 'utm_campaigns'}, +} + +class UTMSources extends React.Component { + constructor(props) { + super(props) + this.state = {loading: true} + } + + componentDidMount() { + this.fetchReferrers() + if (this.props.timer) this.props.timer.onTick(this.fetchReferrers.bind(this)) + } + + componentDidUpdate(prevProps) { + if (this.props.query !== prevProps.query || this.props.tab !== prevProps.tab) { + this.setState({loading: true, referrers: null}) + this.fetchReferrers() + } + } + + showNoRef() { + return this.props.query.period === 'realtime' + } + + fetchReferrers() { + const endpoint = UTM_TAGS[this.props.tab].endpoint + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/${endpoint}`, this.props.query, {show_noref: this.showNoRef()}) + .then((res) => this.setState({loading: false, referrers: res})) + } + + renderReferrer(referrer) { + const query = new URLSearchParams(window.location.search) + query.set(this.props.tab, referrer.name) + + return ( +
    +
    + + + + { referrer.name } + + +
    + {numberFormatter(referrer.count)} +
    + ) + } + + label() { + return this.props.query.period === 'realtime' ? 'Current visitors' : 'Visitors' + } + + renderList() { + if (this.state.referrers.length > 0) { + return ( + +
    + {UTM_TAGS[this.props.tab].label} + {this.label()} +
    + + + {this.state.referrers.map(this.renderReferrer.bind(this))} + +     + ) + } else { + return
    No data yet
    + } + } + + renderContent() { + if (this.state.referrers) { + return ( + +
    +

    Top sources

    + { this.props.renderTabs() } +
    + { this.renderList() } + +     + ) + } + } + + render() { + return ( +
    + { this.state.loading &&
    } + + { this.renderContent() } + +
    + ) + } +} + +export default class SourceList extends React.Component { + constructor(props) { + super(props) + this.tabKey = 'sourceTab__' + props.site.domain + const storedTab = window.localStorage[this.tabKey] + this.state = { + tab: storedTab || 'all' + } + } + + setTab(tab) { + return () => { + window.localStorage[this.tabKey] = tab + this.setState({tab}) + } + } + + renderTabs() { + const activeClass = 'inline-block h-5 text-indigo-700 font-bold border-b-2 border-indigo-700' + const defaultClass = 'hover:text-indigo-700 cursor-pointer' + return ( +
      +
    • All
      • +
    • Medium
      • +
    • Source
      • +
    • Campaign
      • +
    + ) + } + + render() { + if (this.state.tab === 'all') { + return + } else if (this.state.tab === 'utm_medium') { + return + } else if (this.state.tab === 'utm_source') { + return + } else if (this.state.tab === 'utm_campaign') { + return + } + } +} diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index b85b5e9fc..d3179d883 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -194,7 +194,7 @@ defmodule Plausible.Stats.Clickhouse do end) end - def top_referrers(site, query, limit, page, show_noref \\ false, include \\ []) do + def top_sources(site, query, limit, page, show_noref \\ false, include \\ []) do offset = (page - 1) * limit referrers = @@ -203,7 +203,7 @@ defmodule Plausible.Stats.Clickhouse do order_by: [desc: fragment("count"), asc: fragment("min(start)")], limit: ^limit, offset: ^offset - ) + ) |> filter_converted_sessions(site, query) referrers = if show_noref do referrers @@ -247,6 +247,102 @@ defmodule Plausible.Stats.Clickhouse do end) end + defp filter_converted_sessions(db_query, site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do + converted_sessions = + from(e in base_query(site, query), + select: %{session_id: e.session_id}) + + from(s in db_query, + join: cs in subquery(converted_sessions), + on: s.session_id == cs.session_id + ) + end + defp filter_converted_sessions(db_query, _site, _query), do: db_query + + def utm_mediums(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do + offset = (page - 1) * limit + + q = from( + s in base_session_query(site, query), + group_by: s.utm_medium, + order_by: [desc: fragment("count"), asc: fragment("min(start)")], + limit: ^limit, + offset: ^offset, + select: %{ + name: fragment("if(empty(?), ?, ?) as name", s.utm_medium, @no_ref, s.utm_medium), + count: fragment("uniq(user_id) as count"), + bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), + visit_duration: fragment("round(avg(duration * sign))") + } + ) + + q = if show_noref do + q + else + from(s in q, where: s.utm_medium != "") + end + + q + |> filter_converted_sessions(site, query) + |> ClickhouseRepo.all() + end + + def utm_campaigns(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do + offset = (page - 1) * limit + + q = from( + s in base_session_query(site, query), + group_by: s.utm_campaign, + order_by: [desc: fragment("count"), asc: fragment("min(start)")], + limit: ^limit, + offset: ^offset, + select: %{ + name: fragment("if(empty(?), ?, ?) as name", s.utm_campaign, @no_ref, s.utm_campaign), + count: fragment("uniq(user_id) as count"), + bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), + visit_duration: fragment("round(avg(duration * sign))") + } + ) + + q = if show_noref do + q + else + from(s in q, where: s.utm_campaign != "") + end + + q + |> filter_converted_sessions(site, query) + |> ClickhouseRepo.all() + end + + def utm_sources(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do + offset = (page - 1) * limit + + q = from( + s in base_session_query(site, query), + group_by: s.utm_source, + order_by: [desc: fragment("count"), asc: fragment("min(start)")], + limit: ^limit, + offset: ^offset, + select: %{ + name: fragment("if(empty(?), ?, ?) as name", s.utm_source, @no_ref, s.utm_source), + count: fragment("uniq(user_id) as count"), + bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), + visit_duration: fragment("round(avg(duration * sign))") + } + ) + + q = if show_noref do + q + else + from(s in q, where: s.utm_source != "") + end + + q + |> filter_converted_sessions(site, query) + |> ClickhouseRepo.all() + end + def conversions_from_referrer(site, query, referrer) do converted_sessions = from( @@ -543,43 +639,16 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.filter(& &1) if Enum.count(events) > 0 do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - - q = - from( - e in "events", - where: e.domain == ^site.domain, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, - where: fragment("? IN tuple(?)", e.name, ^events), - group_by: e.name, - select: %{ - name: e.name, - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") - } - ) - - q = - if query.filters["source"] do - filtered_sessions = - from(s in base_session_query(site, query), select: %{session_id: s.session_id}) - - from( - e in q, - join: cs in subquery(filtered_sessions), - on: e.session_id == cs.session_id - ) - else - q - end - - q = - if query.filters["page"] do - page = query.filters["page"] - from(e in q, where: e.pathname == ^page) - else - q - end + q = from( + e in base_query_w_sessions(site, query), + where: fragment("? IN tuple(?)", e.name, ^events), + group_by: e.name, + select: %{ + name: e.name, + count: fragment("uniq(user_id) as count"), + total_count: fragment("count(*) as total_count") + } + ) ClickhouseRepo.all(q) else @@ -593,43 +662,16 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.filter(& &1) if Enum.count(pages) > 0 do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - - q = - from( - e in "events", - where: e.domain == ^site.domain, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, - where: fragment("? IN tuple(?)", e.pathname, ^pages), - group_by: e.pathname, - select: %{ - name: fragment("concat('Visit ', ?) as name", e.pathname), - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") - } - ) - - q = - if query.filters["source"] do - filtered_sessions = - from(s in base_session_query(site, query), select: %{session_id: s.session_id}) - - from( - e in q, - join: cs in subquery(filtered_sessions), - on: e.session_id == cs.session_id - ) - else - q - end - - q = - if query.filters["page"] do - page = query.filters["page"] - from(e in q, where: e.pathname == ^page) - else - q - end + q = from( + e in base_query_w_sessions(site, query), + where: fragment("? IN tuple(?)", e.pathname, ^pages), + group_by: e.pathname, + select: %{ + name: fragment("concat('Visit ', ?) as name", e.pathname), + count: fragment("uniq(user_id) as count"), + total_count: fragment("count(*) as total_count") + } + ) ClickhouseRepo.all(q) else @@ -659,6 +701,30 @@ defmodule Plausible.Stats.Clickhouse do sessions_q end + sessions_q = + if query.filters["utm_medium"] do + utm_medium = query.filters["utm_medium"] + from(s in sessions_q, where: s.utm_medium == ^utm_medium) + else + sessions_q + end + + sessions_q = + if query.filters["utm_source"] do + utm_source = query.filters["utm_source"] + from(s in sessions_q, where: s.utm_source == ^utm_source) + else + sessions_q + end + + sessions_q = + if query.filters["utm_campaign"] do + utm_campaign = query.filters["utm_campaign"] + from(s in sessions_q, where: s.utm_campaign == ^utm_campaign) + else + sessions_q + end + sessions_q = if query.filters["referrer"] do ref = query.filters["referrer"] from(s in sessions_q, where: s.referrer == ^ref) @@ -672,7 +738,7 @@ defmodule Plausible.Stats.Clickhouse do where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) - q = if query.filters["source"] || query.filters['referrer'] do + q = if query.filters["source"] || query.filters['referrer'] || query.filters["utm_medium"] || query.filters["utm_source"] || query.filters["utm_campaign"] do from( e in q, join: sq in subquery(sessions_q), @@ -708,6 +774,30 @@ defmodule Plausible.Stats.Clickhouse do q end + q = + if query.filters["utm_medium"] do + utm_medium = query.filters["utm_medium"] + from(s in q, where: s.utm_medium == ^utm_medium) + else + q + end + + q = + if query.filters["utm_source"] do + utm_source = query.filters["utm_source"] + from(s in q, where: s.utm_source == ^utm_source) + else + q + end + + q = + if query.filters["utm_campaign"] do + utm_campaign = query.filters["utm_campaign"] + from(s in q, where: s.utm_campaign == ^utm_campaign) + else + q + end + q = if query.filters["page"] do page = query.filters["page"] @@ -718,7 +808,7 @@ defmodule Plausible.Stats.Clickhouse do if query.filters["referrer"] do ref = query.filters["referrer"] - from(e in q, where: e.referrer == ^ref) + from(s in q, where: s.referrer == ^ref) else q end @@ -743,6 +833,30 @@ defmodule Plausible.Stats.Clickhouse do q end + q = + if query.filters["utm_medium"] do + utm_medium = query.filters["utm_medium"] + from(e in q, where: e.utm_medium == ^utm_medium) + else + q + end + + q = + if query.filters["utm_source"] do + utm_source = query.filters["utm_source"] + from(e in q, where: e.utm_source == ^utm_source) + else + q + end + + q = + if query.filters["utm_campaign"] do + utm_campaign = query.filters["utm_campaign"] + from(e in q, where: e.utm_campaign == ^utm_campaign) + else + q + end + q = if query.filters["referrer"] do ref = query.filters["referrer"] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 8750f5636..75d7b43c0 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -125,23 +125,41 @@ defmodule PlausibleWeb.Api.StatsController do end end - def referrers(conn, params) do + def sources(conn, params) do site = conn.assigns[:site] query = Query.from(site.timezone, params) include = if params["include"], do: String.split(params["include"], ","), else: [] limit = if params["limit"], do: String.to_integer(params["limit"]) page = if params["page"], do: String.to_integer(params["page"]) show_noref = params["show_noref"] == "true" - json(conn, Stats.top_referrers(site, query, limit || 9, page || 1, show_noref, include)) + json(conn, Stats.top_sources(site, query, limit || 9, page || 1, show_noref, include)) end - def referrers_for_goal(conn, params) do + def utm_mediums(conn, params) do site = conn.assigns[:site] query = Query.from(site.timezone, params) limit = if params["limit"], do: String.to_integer(params["limit"]) page = if params["page"], do: String.to_integer(params["page"]) + show_noref = params["show_noref"] == "true" + json(conn, Stats.utm_mediums(site, query, limit || 9, page || 1, show_noref)) + end - json(conn, Stats.top_referrers_for_goal(site, query, limit || 9, page || 1)) + def utm_campaigns(conn, params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, params) + limit = if params["limit"], do: String.to_integer(params["limit"]) + page = if params["page"], do: String.to_integer(params["page"]) + show_noref = params["show_noref"] == "true" + json(conn, Stats.utm_campaigns(site, query, limit || 9, page || 1, show_noref)) + end + + def utm_sources(conn, params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, params) + limit = if params["limit"], do: String.to_integer(params["limit"]) + page = if params["page"], do: String.to_integer(params["page"]) + show_noref = params["show_noref"] == "true" + json(conn, Stats.utm_sources(site, query, limit || 9, page || 1, show_noref)) end @google_api Application.fetch_env!(:plausible, :google_api) diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 3901cc98e..b489367d4 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -46,8 +46,10 @@ defmodule PlausibleWeb.Router do get "/:domain/current-visitors", StatsController, :current_visitors get "/:domain/main-graph", StatsController, :main_graph - get "/:domain/referrers", StatsController, :referrers - get "/:domain/goal/referrers", StatsController, :referrers_for_goal + get "/:domain/sources", StatsController, :sources + get "/:domain/utm_mediums", StatsController, :utm_mediums + get "/:domain/utm_sources", StatsController, :utm_sources + get "/:domain/utm_campaigns", StatsController, :utm_campaigns get "/:domain/referrers/:referrer", StatsController, :referrer_drilldown get "/:domain/goal/referrers/:referrer", StatsController, :referrer_drilldown_for_goal get "/:domain/pages", StatsController, :pages diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 47df0a25e..0bbd544ad 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendEmailReport do bounce_rate = Stats.bounce_rate(site, query) prev_bounce_rate = Stats.bounce_rate(site, Query.shift_back(query)) change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate - referrers = Stats.top_referrers(site, query, 5, 1, []) + referrers = Stats.top_sources(site, query, 5, 1, []) pages = Stats.top_pages(site, query, 5, []) user = Plausible.Auth.find_user_by(email: email) login_link = user && Plausible.Sites.is_owner?(user.id, site) diff --git a/mix.lock b/mix.lock index 5f9796b16..d427a1d35 100644 --- a/mix.lock +++ b/mix.lock @@ -5,7 +5,7 @@ "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.2.0", "3df902b81ce7fa8867a2ae30d20a1da6877a2c056bfb116fd0bc8a5f0190cea4", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "762be3fcb779f08207531bc6612cca480a338e4b4357abb49f5ce00240a77d1e"}, "browser": {:hex, :browser, "0.4.4", "bd6436961a6b2299c6cb38d0e49761c1161d869cd0db46369cef2bf6b77c3665", [:mix], [{:plug, "~> 1.2", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d476ca309d4a4b19742b870380390aabbcb323c1f6f8745e2da2dfd079b4f8d7"}, "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, - "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "bd644593820c6e9c19f25e2cedc8891c5a57f60f", []}, + "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "2d79f932bdb7f8894668c650b9dcc124d2145aeb", []}, "clickhousex": {:git, "https://github.com/plausible/clickhousex", "89d58d4cb0cad2558e874f30e81a5c2c84ada95e", []}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.1", "7fe612b739c78c9c1a75186ef2d322ce4d25032d119823269d0aa1e2f1e20025", [:mix], [], "hexpm", "d6222483060c17f0977fad1b7401ef0c5863c985a64352755f366aee3799c245"}, diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs similarity index 78% rename from test/plausible_web/controllers/api/stats_controller/referrers_test.exs rename to test/plausible_web/controllers/api/stats_controller/sources_test.exs index 8c20deb63..caa2e0e67 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -1,12 +1,12 @@ -defmodule PlausibleWeb.Api.StatsController.ReferrersTest do +defmodule PlausibleWeb.Api.StatsController.SourcesTest do use PlausibleWeb.ConnCase import Plausible.TestUtils - describe "GET /api/stats/:domain/referrers" do + describe "GET /api/stats/:domain/sources" do setup [:create_user, :log_in, :create_site] - test "returns top referrer sources by user ids", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01") + test "returns top sources by unique user ids", %{conn: conn, site: site} do + conn = get(conn, "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01") assert json_response(conn, 200) == [ %{"name" => "10words", "count" => 2, "url" => "10words.com"}, @@ -14,11 +14,11 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ] end - test "calculates bounce rate and visit duration for referrers", %{conn: conn, site: site} do + test "calculates bounce rate and visit duration for sources", %{conn: conn, site: site} do conn = get( conn, - "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate,visit_duration" + "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&include=bounce_rate,visit_duration" ) assert json_response(conn, 200) == [ @@ -39,8 +39,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ] end - test "returns top referrer sources in realtime report", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/referrers?period=realtime") + test "returns top sources in realtime report", %{conn: conn, site: site} do + conn = get(conn, "/api/stats/#{site.domain}/sources?period=realtime") assert json_response(conn, 200) == [ %{"name" => "10words", "count" => 2, "url" => "10words.com"}, @@ -49,7 +49,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do end test "can paginate the results", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&limit=1&page=2") + conn = get(conn, "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&limit=1&page=2") assert json_response(conn, 200) == [ %{"name" => "Bing", "count" => 1, "url" => ""} @@ -57,6 +57,18 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do end end + describe "GET /api/stats/:domain/utm_mediums" do + setup [:create_user, :log_in, :create_site] + + test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do + conn = get(conn, "/api/stats/#{site.domain}/utm_mediums?period=day&date=2019-01-01") + + assert json_response(conn, 200) == [ + %{"name" => "listing", "count" => 2, "bounce_rate" => 50.0, "visit_duration" => 50.0}, + %{"name" => "search", "count" => 1, "bounce_rate" => 0.0, "visit_duration" => 100.0} + ] + end + end describe "GET /api/stats/:domain/goal/referrers" do setup [:create_user, :log_in, :create_site] @@ -67,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get( conn, - "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}" + "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&filters=#{filters}" ) assert json_response(conn, 200) == [ @@ -81,7 +93,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get( conn, - "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}" + "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&filters=#{filters}" ) assert json_response(conn, 200) == [ @@ -98,7 +110,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{filters}") assert json_response(conn, 200) == %{ - "total_visitors" => 2, + "total_visitors" => 6, "referrers" => [ %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2} ] @@ -114,7 +126,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ) assert json_response(conn, 200) == %{ - "total_visitors" => 2, + "total_visitors" => 6, "referrers" => [ %{ "name" => "10words.com/page1", diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex index d25414dc8..63ab33a77 100644 --- a/test/support/clickhouse_setup.ex +++ b/test/support/clickhouse_setup.ex @@ -147,6 +147,7 @@ defmodule Plausible.Test.ClickhouseSetup do exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", + utm_medium: "listing", session_id: @conversion_1_session_id, is_bounce: true, duration: 100, @@ -159,6 +160,7 @@ defmodule Plausible.Test.ClickhouseSetup do exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", + utm_medium: "listing", session_id: @conversion_2_session_id, is_bounce: false, duration: 0, @@ -171,6 +173,7 @@ defmodule Plausible.Test.ClickhouseSetup do exit_page: "/", referrer_source: "Bing", referrer: "", + utm_medium: "search", is_bounce: false, duration: 100, start: ~N[2019-01-01 03:00:00],