Page drilldown (#276)

* WIP

* Add filter option for page

* Use JOIN to get correct top numbers

* Filter google search terms by page

* Allow filtering by page from modal

* More robust base url for google search console integration

* Fix tests

assets/js/dashboard/filters.js

@@ -12,6 +12,9 @@ function filterText(key, value) {
   if (key === "referrer") {
     return <span className="inline-block max-w-sm truncate">Referrer: <b>{value}</b></span>
   }
+  if (key === "page") {
+    return <span className="inline-block max-w-sm truncate">Page: <b>{value}</b></span>
+  }
 }
 
 function renderFilter(history, [key, value]) {

assets/js/dashboard/query.js

@@ -25,7 +25,8 @@ export function parseQuery(querystring, site) {
     filters: {
       'goal': q.get('goal'),
       'source': q.get('source'),
-      'referrer': q.get('referrer')
+      'referrer': q.get('referrer'),
+      'page': q.get('page')
     }
   }
 }

assets/js/dashboard/stats/modals/pages.js

@@ -1,4 +1,5 @@
 import React from "react";
+import { Link } from 'react-router-dom'
 import { withRouter } from 'react-router-dom'
 
 import Modal from './modal'
@@ -18,8 +19,14 @@ class PagesModal extends React.Component {
   componentDidMount() {
     const include = this.showBounceRate() ? 'bounce_rate' : null
 
-    api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, this.state.query, {limit: 100, include: include})
-      .then((res) => this.setState({loading: false, pages: res}))
+    const {filters} = this.state.query
+    if (filters.source || filters.referrer) {
+      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/entry-pages`, this.state.query, {limit: 100, include: include})
+        .then((res) => this.setState({loading: false, pages: res}))
+    } else {
+      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, this.state.query, {limit: 100, include: include})
+        .then((res) => this.setState({loading: false, pages: res}))
+    }
   }
 
   showBounceRate() {
@@ -40,9 +47,14 @@ class PagesModal extends React.Component {
   }
 
   renderPage(page) {
+    const query = new URLSearchParams(window.location.search)
+    query.set('page', page.name)
+
     return (
       <tr className="text-sm" key={page.name}>
-        <td className="p-2 truncate">{page.name}</td>
+        <td className="p-2 truncate">
+          <Link to={{pathname: `/${encodeURIComponent(this.props.site.domain)}`, search: query.toString()}} className="hover:underline">{page.name}</Link>
+        </td>
         <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.count)}</td>
         {this.showPageviews() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.pageviews)}</td> }
         {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(page)}</td> }

assets/js/dashboard/stats/more-link.js

@@ -4,7 +4,7 @@ import { Link } from 'react-router-dom'
 export default function MoreLink({site, list, endpoint}) {
   if (list.length > 0) {
     return (
-      <div className="text-center w-full absolute bottom-0 left-0 p-4">
+      <div className="text-center w-full absolute bottom-0 left-0 pb-3">
         <Link to={`/${encodeURIComponent(site.domain)}/${endpoint}${window.location.search}`} className="leading-snug font-bold text-sm text-gray-500 hover:text-red-500 transition tracking-wide">
           <svg className="feather mr-1" style={{marginTop: '-2px'}} xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"/></svg>
           MORE

assets/js/dashboard/stats/pages.js

@@ -1,4 +1,5 @@
 import React from 'react';
+import { Link } from 'react-router-dom'
 import FlipMove from 'react-flip-move';
 
 import FadeIn from '../fade-in'
@@ -38,11 +39,14 @@ export default class Pages extends React.Component {
   }
 
   renderPage(page) {
+    const query = new URLSearchParams(window.location.search)
+    query.set('page', page.name)
+
     return (
       <div className="flex items-center justify-between my-1 text-sm" key={page.name}>
         <div className="w-full h-8 truncate" style={{maxWidth: 'calc(100% - 4rem)'}}>
           <Bar count={page.count} all={this.state.pages} bg="bg-orange-50" />
-          <span className="block px-2" style={{marginTop: '-26px'}}>{page.name}</span>
+          <Link to={{search: query.toString()}} className="block px-2 hover:underline" style={{marginTop: '-26px'}}>{page.name}</Link>
         </div>
         <span className="font-medium">{numberFormatter(page.count)}</span>
       </div>
@@ -50,7 +54,14 @@ export default class Pages extends React.Component {
   }
 
   label() {
-    return this.props.query.period === 'realtime' ? 'Active visitors' : 'Visitors'
+    const filters = this.props.query.filters
+    if (this.props.query.period === 'realtime') {
+      return 'Active visitors'
+    } else if (filters['source'] || filters['referrer']) {
+      return 'Entrances'
+    } else {
+      return 'Visitors'
+    }
   }
 
   renderList() {

lib/plausible/google/api.ex

@@ -41,9 +41,23 @@ defmodule Plausible.Google.Api do
     |> Enum.map(fn url -> String.trim_trailing(url, "/") end)
   end
 
-  def fetch_stats(auth, query, limit) do
-    auth = refresh_if_needed(auth)
+  defp property_base_url(property) do
+    case property do
+      "sc-domain:" <> domain -> "https://" <> domain
+        url -> url
+    end
+  end
+
+  def fetch_stats(site, query, limit) do
+    auth = refresh_if_needed(site.google_auth)
     property = URI.encode_www_form(auth.property)
+    base_url = property_base_url(auth.property)
+    filter_groups = if query.filters["page"] do
+      [%{filters: [%{
+        dimension: "page",
+        expression: "https://#{base_url}#{query.filters["page"]}"
+      }]}]
+    end
 
     res =
       HTTPoison.post!(
@@ -52,10 +66,11 @@ defmodule Plausible.Google.Api do
           startDate: Date.to_iso8601(query.date_range.first),
           endDate: Date.to_iso8601(query.date_range.last),
           dimensions: ["query"],
-          rowLimit: limit
+          rowLimit: limit,
+          dimensionFilterGroups: filter_groups || %{}
         }),
         "Content-Type": "application/json",
-        Authorization: "Bearer #{auth.access_token}"
+        Authorization: "Bearer #{site.google_auth.access_token}"
       )
 
     case res.status_code do

lib/plausible/stats/clickhouse.ex

@@ -177,9 +177,9 @@ defmodule Plausible.Stats.Clickhouse do
   def pageviews_and_visitors(site, query) do
     [res] =
       Clickhouse.all(
-        from e in base_session_query(site, query),
+        from e in base_query_w_sessions(site, query),
           select:
-            {fragment("sum(sign * pageviews) as pageviews"),
+            {fragment("count(*) as pageviews"),
              fragment("uniq(user_id) as visitors")}
       )
 
@@ -234,6 +234,13 @@ defmodule Plausible.Stats.Clickhouse do
       from(s in referrers, where: s.referrer_source != "")
     end
 
+    referrers = if query.filters["page"] do
+      page = query.filters["page"]
+      from(s in referrers, where: s.entry_page == ^page)
+    else
+      referrers
+    end
+
     referrers =
       if "bounce_rate" in include do
         from(
@@ -351,8 +358,8 @@ defmodule Plausible.Stats.Clickhouse do
   end
 
   def entry_pages(site, query, limit, include) do
-    pages = Clickhouse.all(
-      from s in base_session_query(site, query),
+    q = from(
+      s in base_session_query(site, query),
       group_by: s.entry_page,
       order_by: [desc: fragment("count")],
       limit: ^limit,
@@ -360,6 +367,15 @@ defmodule Plausible.Stats.Clickhouse do
       {fragment("? as name", s.entry_page), fragment("uniq(?) as count", s.user_id)}
     )
 
+    q = if query.filters["page"] do
+      page = query.filters["page"]
+      from(s in q, where: s.entry_page == ^page)
+    else
+      q
+    end
+
+    pages = Clickhouse.all(q)
+
     if "bounce_rate" in include do
       bounce_rates = bounce_rates_by_page_url(site, query)
       Enum.map(pages, fn url -> Map.put(url, "bounce_rate", bounce_rates[url["name"]]) end)
@@ -562,6 +578,14 @@ defmodule Plausible.Stats.Clickhouse do
           q
         end
 
+      q =
+        if query.filters["page"] do
+          page = query.filters["page"]
+          from(e in q, where: e.pathname == ^page)
+        else
+          q
+        end
+
       Clickhouse.all(q)
     else
       []
@@ -602,6 +626,14 @@ defmodule Plausible.Stats.Clickhouse do
           q
         end
 
+      q =
+        if query.filters["page"] do
+          page = query.filters["page"]
+          from(e in q, where: e.pathname == ^page)
+        else
+          q
+        end
+
       Clickhouse.all(q)
     else
       []
@@ -612,6 +644,55 @@ defmodule Plausible.Stats.Clickhouse do
     Enum.sort_by(conversions, fn conversion -> -conversion["count"] end)
   end
 
+  defp base_query_w_sessions(site, query) do
+    {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)
+
+    sessions_q = from(s in "sessions",
+      where: s.domain == ^site.domain,
+      where: s.timestamp >= ^first_datetime and s.start < ^last_datetime,
+      select: %{session_id: s.session_id}
+    )
+
+    sessions_q =
+      if query.filters["source"] do
+        source = query.filters["source"]
+        source = if source == @no_ref, do: "", else: source
+        from(s in sessions_q, where: s.referrer_source == ^source)
+      else
+        sessions_q
+      end
+
+    sessions_q = if query.filters["referrer"] do
+      ref = query.filters["referrer"]
+      from(s in sessions_q, where: s.referrer == ^ref)
+    else
+      sessions_q
+    end
+
+    q =
+      from(e in "events",
+        where: e.domain == ^site.domain,
+        where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+      )
+
+    q = if query.filters["source"] || query.filters['referrer'] do
+      from(
+        e in q,
+        join: sq in subquery(sessions_q),
+        on: e.session_id == sq.session_id
+      )
+    else
+      q
+    end
+
+    if query.filters["page"] do
+      page = query.filters["page"]
+      from(e in q, where: e.pathname == ^page)
+    else
+      q
+    end
+  end
+
   defp base_session_query(site, query) do
     {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)
 
@@ -625,7 +706,15 @@ defmodule Plausible.Stats.Clickhouse do
       if query.filters["source"] do
         source = query.filters["source"]
         source = if source == @no_ref, do: "", else: source
-        from(e in q, where: e.referrer_source == ^source)
+        from(s in q, where: s.referrer_source == ^source)
+      else
+        q
+      end
+
+    q =
+      if query.filters["page"] do
+        page = query.filters["page"]
+        from(s in q, where: s.entry_page == ^page)
       else
         q
       end
@@ -665,6 +754,14 @@ defmodule Plausible.Stats.Clickhouse do
         q
       end
 
+    q =
+      if query.filters["page"] do
+        page = query.filters["page"]
+        from(e in q, where: e.pathname == ^page)
+      else
+        q
+      end
+
     q =
       if path do
         from(e in q, where: e.pathname == ^path)

lib/plausible_web/controllers/api/stats_controller.ex

@@ -78,8 +78,16 @@ defmodule PlausibleWeb.Api.StatsController do
     bounce_rate = Stats.bounce_rate(site, query)
     prev_bounce_rate = Stats.bounce_rate(site, prev_query)
     change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate
-    visit_duration = Stats.visit_duration(site, query)
-    prev_visit_duration = Stats.visit_duration(site, prev_query)
+    visit_duration = if !query.filters["page"] do
+      duration = Stats.visit_duration(site, query)
+      prev_duration = Stats.visit_duration(site, prev_query)
+
+      %{
+        name: "Visit duration",
+        count: duration,
+        change: percent_change(prev_duration, duration)
+      }
+    end
 
     [
       %{
@@ -93,12 +101,8 @@ defmodule PlausibleWeb.Api.StatsController do
         change: percent_change(prev_pageviews, pageviews)
       },
       %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate},
-      %{
-        name: "Visit duration",
-        count: visit_duration,
-        change: percent_change(prev_visit_duration, visit_duration)
-      }
-    ]
+      visit_duration
+    ] |> Enum.filter(&(&1))
   end
 
   defp percent_change(old_count, new_count) do
@@ -138,7 +142,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
     search_terms =
       if site.google_auth && site.google_auth.property && !query.filters["goal"] do
-        @google_api.fetch_stats(site.google_auth, query, params["limit"] || 9)
+        @google_api.fetch_stats(site, query, params["limit"] || 9)
       end
 
     case search_terms do

test/plausible_web/controllers/api/stats_controller/main_graph_test.exs

@@ -83,14 +83,14 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
       conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")
 
       res = json_response(conn, 200)
-      assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"]
+      assert %{"name" => "Unique visitors", "count" => 9, "change" => 100} in res["top_stats"]
     end
 
     test "counts total pageviews", %{conn: conn, site: site} do
       conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")
 
       res = json_response(conn, 200)
-      assert %{"name" => "Total pageviews", "count" => 3, "change" => 100} in res["top_stats"]
+      assert %{"name" => "Total pageviews", "count" => 9, "change" => 100} in res["top_stats"]
     end
 
     test "calculates bounce rate", %{conn: conn, site: site} do

test/plausible_web/controllers/api/stats_controller/referrers_test.exs

@@ -89,7 +89,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
       conn = get(conn, "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{filters}")
 
       assert json_response(conn, 200) == %{
-               "total_visitors" => 2,
+               "total_visitors" => 6,
                "referrers" => [
                  %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2}
                ]
@@ -105,7 +105,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
         )
 
       assert json_response(conn, 200) == %{
-               "total_visitors" => 2,
+               "total_visitors" => 6,
                "referrers" => [
                  %{
                    "name" => "10words.com/page1",