Filtering Search Console keywords (#4077)

* Apply filters in search console request

* Remove dead code from search console modal

* Remove unimportant information from keyword modal

* Show invalid filters from search console

* Fix tests

* Add/Fix tests

* Fix typo

* Remove unused variable

* Fix typo

* Changelog entry

* Fix Credo

* Display impressions, CTR and position in keyword modal

* Undo change that should not have been committed

* Fix test

* Fix test

* filters -> search_console_filters

CHANGELOG.md

@@ -42,6 +42,7 @@ All notable changes to this project will be documented in this file.
 - Add Yesterday as an time range option in the dashboard
 - Add support for importing Google Analytics 4 data
 - Import custom events from Google Analytics 4
+- Ability to filter Search Console keywords by page, country and device plausible/analytics#4077
 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
 
 ### Removed

assets/js/dashboard/stats/modals/exit-pages.js

@@ -4,7 +4,7 @@ import { withRouter } from 'react-router-dom'
 
 import Modal from './modal'
 import * as api from '../../api'
-import numberFormatter from '../../util/number-formatter'
+import numberFormatter, {percentageFormatter} from '../../util/number-formatter'
 import { parseQuery } from '../../query'
 import { trimURL } from '../../util/url'
 class ExitPagesModal extends React.Component {
@@ -34,14 +34,6 @@ class ExitPagesModal extends React.Component {
     this.setState({ loading: true, page: this.state.page + 1 }, this.loadPages.bind(this))
   }
 
-  formatPercentage(number) {
-    if (typeof (number) === 'number') {
-      return number + '%'
-    } else {
-      return '-'
-    }
-  }
-
   showConversionRate() {
     return !!this.state.query.filters.goal
   }
@@ -74,7 +66,7 @@ class ExitPagesModal extends React.Component {
         {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.total_visitors)}</td>}
         <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visitors)}</td>
         {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visits)}</td>}
-        {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatPercentage(page.exit_rate)}</td>}
+        {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(page.exit_rate)}</td>}
         {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.conversion_rate)}%</td>}
       </tr>
     )

assets/js/dashboard/stats/modals/google-keywords.js

@@ -3,7 +3,7 @@ import { Link, withRouter } from 'react-router-dom'
 
 import Modal from './modal'
 import * as api from '../../api'
-import numberFormatter from '../../util/number-formatter'
+import numberFormatter, { percentageFormatter } from '../../util/number-formatter'
 import {parseQuery} from '../../query'
 import RocketIcon from './rocket-icon'
 
@@ -17,49 +17,31 @@ class GoogleKeywordsModal extends React.Component {
   }
 
   componentDidMount() {
-    if (this.state.query.filters.goal) {
+    api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})
-      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers/Google`, this.state.query, {limit: 100})
+      .then((res) => this.setState({
-        .then((res) => this.setState({
+        loading: false,
-          loading: false,
+        searchTerms: res.search_terms,
-          searchTerms: res.search_terms,
+        notConfigured: res.not_configured,
-          totalVisitors: res.total_visitors,
+        isOwner: res.is_owner
-          notConfigured: res.not_configured,
+      }))
-          isOwner: res.is_owner
-        }))
-    } else {
-      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})
-        .then((res) => this.setState({
-          loading: false,
-          searchTerms: res.search_terms,
-          totalVisitors: res.total_visitors,
-          notConfigured: res.not_configured,
-          isOwner: res.is_owner
-        }))
-    }
   }
 
   renderTerm(term) {
     return (
       <React.Fragment key={term.name}>
-
         <tr className="text-sm dark:text-gray-200" key={term.name}>
-          <td className="p-2 truncate">{term.name}</td>
+          <td className="p-2">{term.name}</td>
           <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.visitors)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.impressions)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(term.ctr)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.position)}</td>
         </tr>
       </React.Fragment>
     )
   }
 
   renderKeywords() {
-    if (this.state.query.filters.goal) {
+    if (this.state.notConfigured) {
-      return (
-        <div className="text-center text-gray-700 dark:text-gray-300 mt-6">
-          <RocketIcon />
-          <div className="text-lg">Sorry, we cannot show which keywords converted best for goal <b>{this.state.query.filters.goal}</b></div>
-          <div className="text-lg">Google does not share this information</div>
-        </div>
-      )
-    } else if (this.state.notConfigured) {
       if (this.state.isOwner) {
         return (
           <div className="text-center text-gray-700 dark:text-gray-300 mt-6">
@@ -84,7 +66,10 @@ class GoogleKeywordsModal extends React.Component {
           <thead>
             <tr>
               <th className="p-2 w-48 md:w-56 lg:w-1/3 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="left">Search Term</th>
-              <th className="p-2 w-32 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Impressions</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">CTR</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Position</th>
             </tr>
           </thead>
           <tbody>
@@ -102,14 +87,6 @@ class GoogleKeywordsModal extends React.Component {
     }
   }
 
-  renderGoalText() {
-    if (this.state.query.filters.goal) {
-      return (
-        <h1 className="text-xl font-semibold text-gray-500 dark:text-gray-200 leading-none">completed {this.state.query.filters.goal}</h1>
-      )
-    }
-  }
-
   renderBody() {
     if (this.state.loading) {
       return (
@@ -122,10 +99,6 @@ class GoogleKeywordsModal extends React.Component {
 
           <div className="my-4 border-b border-gray-300 dark:border-gray-500"></div>
           <main className="modal__content">
-            <h1 className="text-xl font-semibold mb-0 leading-none dark:text-gray-200">
-              {this.state.totalVisitors} visitors from Google<br />
-            </h1>
-            {this.renderGoalText()}
             { this.renderKeywords() }
           </main>
         </React.Fragment>

assets/js/dashboard/stats/sources/search-terms.js

@@ -39,7 +39,8 @@ export default class SearchTerms extends React.Component {
         loading: false,
         searchTerms: res.search_terms || [],
         notConfigured: res.not_configured,
-        isAdmin: res.is_admin
+        isAdmin: res.is_admin,
+        unsupportedFilters: res.unsupported_filters
       })).catch((error) =>
         {
             this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin })
@@ -68,21 +69,19 @@ export default class SearchTerms extends React.Component {
   }
 
   renderList() {
-    if (this.props.query.filters.goal) {
+   if (this.state.unsupportedFilters)  {
       return (
         <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
           <RocketIcon />
-          <div>Sorry, we cannot show which keywords converted best for goal <b>{this.props.query.filters.goal}</b></div>
+          <div>Unable to fetch keyword data from Search Console because it does not support the current set of filters</div>
-          <div>Google does not share this information</div>
         </div>
       )
-
     } else if (this.state.notConfigured) {
       return (
         <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
           <RocketIcon />
           <div>
-          This site is not connected to Search Console so we cannot show the search phrases.
+          This site is not connected to Search Console so we cannot show the search terms
           {this.state.isAdmin && this.state.error && <><br/><br/><p>Please click below to connect your Search Console account.</p></>}
           </div>
           {this.state.isAdmin && <a href={`/${encodeURIComponent(this.props.site.domain)}/settings/integrations`} className="button mt-4">Connect with Google</a> }
@@ -103,9 +102,8 @@ export default class SearchTerms extends React.Component {
       )
     } else {
       return (
-        <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
+        <div className="text-center text-gray-700 dark:text-gray-300 ">
-          <RocketIcon />
+          <div className="mt-44 mx-auto font-medium text-gray-500 dark:text-gray-400">No data yet</div>
-          <div>No search terms were found for this period. Please adjust or extend your time range. Check <a href="https://plausible.io/docs/google-search-console-integration#i-dont-see-google-search-query-data-in-my-dashboard" target="_blank" rel="noreferrer" className="hover:underline text-indigo-700 dark:text-indigo-500">our documentation</a> for more details.</div>
         </div>
       )
     }

assets/js/dashboard/util/number-formatter.js

@@ -49,3 +49,11 @@ export function durationFormatter(duration) {
     return `${seconds}s`
   }
 }
+
+export function percentageFormatter(number) {
+  if (typeof (number) === 'number') {
+    return number + '%'
+  } else {
+    return '-'
+  }
+}

fixture/http_mocks/google_analytics_stats#without_page.json

@@ -1,41 +0,0 @@
-[
-  {
-    "status": 200,
-    "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",
-    "method": "post",
-    "request_body": {
-      "dimensionFilterGroups": {},
-      "dimensions": [
-        "query"
-      ],
-      "endDate": "2022-01-05",
-      "rowLimit": 5,
-      "startDate": "2022-01-01"
-    },
-    "response_body": {
-      "responseAggregationType": "auto",
-      "rows": [
-        {
-          "clicks": 25.0,
-          "ctr": 0.3,
-          "impressions": 50.0,
-          "keys": [
-            "keyword1",
-            "keyword2"
-          ],
-          "position": 2.0
-        },
-        {
-          "clicks": 15.0,
-          "ctr": 0.5,
-          "impressions": 25.0,
-          "keys": [
-            "keyword3",
-            "keyword4"
-          ],
-          "position": 4.0
-        }
-      ]
-    }
-  }
-]

fixture/http_mocks/google_analytics_stats.json

@@ -4,7 +4,7 @@
     "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",
     "method": "post",
     "request_body": {
-      "dimensionFilterGroups": {},
+      "dimensionFilterGroups": [],
       "dimensions": [
         "query"
       ],
@@ -16,26 +16,20 @@
       "responseAggregationType": "auto",
       "rows": [
         {
-          "clicks": 25.0,
+          "clicks": 25,
-          "ctr": 0.3,
+          "ctr": 0.3679,
-          "impressions": 50.0,
+          "impressions": 50,
-          "keys": [
+          "keys": ["keyword1"],
-            "keyword1",
+          "position": 2.2312312
-            "keyword2"
-          ],
-          "position": 2.0
         },
         {
-          "clicks": 15.0,
+          "clicks": 15,
           "ctr": 0.5,
-          "impressions": 25.0,
+          "impressions": 25,
-          "keys": [
+          "keys": ["keyword3"],
-            "keyword3",
-            "keyword4"
-          ],
           "position": 4.0
         }
       ]
     }
   }
-]
+]

lib/plausible/google/api.ex

@@ -6,6 +6,7 @@ defmodule Plausible.Google.API do
   use Timex
 
   alias Plausible.Google.HTTP
+  alias Plausible.Google.SearchConsole
 
   require Logger
 
@@ -74,21 +75,26 @@ defmodule Plausible.Google.API do
   end
 
   def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
-    with site <- Plausible.Repo.preload(site, :google_auth),
+    with {:ok, site} <- ensure_search_console_property(site),
          {:ok, access_token} <- maybe_refresh_token(site.google_auth),
+         {:ok, search_console_filters} <-
+           SearchConsole.Filters.transform(site.google_auth.property, filters),
          {:ok, stats} <-
            HTTP.list_stats(
              access_token,
              site.google_auth.property,
              date_range,
              limit,
-             filters["page"]
+             search_console_filters
            ) do
       stats
       |> Map.get("rows", [])
-      |> Enum.filter(fn row -> row["clicks"] > 0 end)
+      |> Enum.map(&search_console_row/1)
-      |> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)
       |> then(&{:ok, &1})
+    else
+      :google_property_not_configured -> {:error, :google_property_not_configured}
+      :unsupported_filters -> {:error, :unsupported_filters}
+      {:error, error} -> {:error, error}
     end
   end
 
@@ -142,6 +148,44 @@ defmodule Plausible.Google.API do
     Timex.before?(expires_at, thirty_seconds_ago)
   end
 
+  defp ensure_search_console_property(site) do
+    site = Plausible.Repo.preload(site, :google_auth)
+
+    if site.google_auth && site.google_auth.property do
+      {:ok, site}
+    else
+      :google_property_not_configured
+    end
+  end
+
+  defp search_console_row(row) do
+    %{
+      # We always request just one dimension at a time (`query`)
+      name: row["keys"] |> List.first(),
+      visitors: row["clicks"],
+      impressions: row["impressions"],
+      ctr: rounded_ctr(row["ctr"]),
+      position: rounded_position(row["position"])
+    }
+  end
+
+  defp rounded_ctr(ctr) do
+    {:ok, decimal} = Decimal.cast(ctr)
+
+    decimal
+    |> Decimal.mult(100)
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
+  defp rounded_position(position) do
+    {:ok, decimal} = Decimal.cast(position)
+
+    decimal
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
   defp client_id() do
     Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
   end

lib/plausible/google/http.ex

@@ -39,26 +39,18 @@ defmodule Plausible.Google.HTTP do
     response.body
   end
 
-  def list_stats(access_token, property, date_range, limit, page \\ nil) do
+  def list_stats(access_token, property, date_range, limit, search_console_filters) do
-    property = URI.encode_www_form(property)
-
-    filter_groups =
-      if page do
-        url = property_base_url(property)
-        [%{filters: [%{dimension: "page", expression: "https://#{url}#{page}"}]}]
-      else
-        %{}
-      end
-
     params = %{
       startDate: Date.to_iso8601(date_range.first),
       endDate: Date.to_iso8601(date_range.last),
       dimensions: ["query"],
       rowLimit: limit,
-      dimensionFilterGroups: filter_groups
+      dimensionFilterGroups: search_console_filters
     }
 
-    url = "#{api_url()}/webmasters/v3/sites/#{property}/searchAnalytics/query"
+    url =
+      "#{api_url()}/webmasters/v3/sites/#{URI.encode_www_form(property)}/searchAnalytics/query"
+
     headers = [{"Authorization", "Bearer #{access_token}"}]
 
     case HTTPClient.impl().post(url, headers, params) do
@@ -78,9 +70,6 @@ defmodule Plausible.Google.HTTP do
     end
   end
 
-  defp property_base_url("sc-domain:" <> domain), do: "https://" <> domain
-  defp property_base_url(url), do: url
-
   def refresh_auth_token(refresh_token) do
     url = "#{api_url()}/oauth2/v4/token"
     headers = [{"content-type", "application/x-www-form-urlencoded"}]

lib/plausible/google/search_console/filters.ex

@@ -0,0 +1,83 @@
+defmodule Plausible.Google.SearchConsole.Filters do
+  @moduledoc false
+  import Plausible.Stats.Base, only: [page_regex: 1]
+
+  def transform(property, plausible_filters) do
+    plausible_filters = Map.drop(plausible_filters, ["visit:source"])
+
+    search_console_filters =
+      Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->
+        case transform_filter(property, plausible_filter) do
+          :unsupported -> {:halt, :unsupported_filters}
+          search_console_filter -> {:cont, [search_console_filter | search_console_filters]}
+        end
+      end)
+
+    case search_console_filters do
+      :unsupported_filters -> :unsupported_filters
+      [] -> {:ok, []}
+      filters when is_list(filters) -> {:ok, [%{filters: filters}]}
+    end
+  end
+
+  defp transform_filter(property, {"event:page", filter}) do
+    transform_filter(property, {"visit:entry_page", filter})
+  end
+
+  defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do
+    %{dimension: "page", expression: property_url(property, page)}
+  end
+
+  defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do
+    expression =
+      Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
+
+    %{dimension: "page", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do
+    page = page_regex(property_url(property, page))
+    %{dimension: "page", operator: "includingRegex", expression: page}
+  end
+
+  defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})
+       when is_list(pages) do
+    expression =
+      Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
+
+    %{dimension: "page", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do
+    %{dimension: "device", expression: search_console_device(device)}
+  end
+
+  defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do
+    expression = devices |> Enum.join("|")
+    %{dimension: "device", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do
+    %{dimension: "country", expression: search_console_country(country)}
+  end
+
+  defp transform_filter(_property, {"visit:country", {:member, countries}})
+       when is_list(countries) do
+    expression = Enum.map_join(countries, "|", &search_console_country/1)
+    %{dimension: "country", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_, _filter), do: :unsupported
+
+  defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page
+  defp property_url(url, page), do: url <> page
+
+  defp search_console_device("Desktop"), do: "DESKTOP"
+  defp search_console_device("Mobile"), do: "MOBILE"
+  defp search_console_device("Tablet"), do: "TABLET"
+
+  defp search_console_country(alpha_2) do
+    country = Location.Country.get_country(alpha_2)
+    country.alpha_3
+  end
+end

lib/plausible_web/controllers/api/stats_controller.ex

@@ -680,36 +680,29 @@ defmodule PlausibleWeb.Api.StatsController do
   end
 
   def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do
-    site = conn.assigns[:site] |> Repo.preload(:google_auth)
+    site = conn.assigns[:site]
 
-    query =
+    query = Query.from(site, params)
-      Query.from(site, params)
-      |> Query.put_filter("visit:source", "Google")
-
-    search_terms =
-      if site.google_auth && site.google_auth.property && !query.filters["goal"] do
-        google_api().fetch_stats(site, query, params["limit"] || 9)
-      end
-
-    %{:visitors => %{value: total_visitors}} = Stats.aggregate(site, query, [:visitors])
 
     user_id = get_session(conn, :current_user_id)
     is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site)
 
-    case search_terms do
+    case google_api().fetch_stats(site, query, params["limit"] || 9) do
-      nil ->
+      {:error, :google_propery_not_configured} ->
-        json(conn, %{not_configured: true, is_admin: is_admin, total_visitors: total_visitors})
+        json(conn, %{not_configured: true, is_admin: is_admin})
+
+      {:error, :unsupported_filters} ->
+        json(conn, %{unsupported_filters: true})
 
       {:ok, terms} ->
-        json(conn, %{search_terms: terms, total_visitors: total_visitors})
+        json(conn, %{search_terms: terms})
 
       {:error, _} ->
         conn
         |> put_status(502)
         |> json(%{
           not_configured: true,
-          is_admin: is_admin,
+          is_admin: is_admin
-          total_visitors: total_visitors
         })
     end
   end

test/plausible/google/api_test.exs

@@ -245,7 +245,7 @@ defmodule Plausible.Google.APITest do
           "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",
           [{"Authorization", "Bearer 123"}],
           %{
-            dimensionFilterGroups: %{},
+            dimensionFilterGroups: [],
             dimensions: ["query"],
             endDate: "2022-01-05",
             rowLimit: 5,
@@ -301,82 +301,96 @@ defmodule Plausible.Google.APITest do
     end
   end
 
-  describe "fetch_stats/3 with VCR cassetes" do
+  test "returns error when token refresh fails", %{user: user, site: site} do
-    test "returns name and visitor count", %{user: user, site: site} do
+    mock_http_with("google_analytics_auth#invalid_grant.json")
+
+    insert(:google_auth,
+      user: user,
+      site: site,
+      property: "sc-domain:dummy.test",
+      access_token: "*****",
+      refresh_token: "*****",
+      expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600)
+    )
+
+    query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+
+    assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)
+  end
+
+  test "returns error when google auth not configured", %{site: site} do
+    query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+
+    assert {:error, :google_property_not_configured} = Google.API.fetch_stats(site, query, 5)
+  end
+
+  describe "fetch_stats/3 with valid auth" do
+    setup %{user: user, site: site} do
+      insert(:google_auth,
+        user: user,
+        site: site,
+        property: "sc-domain:dummy.test",
+        expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
+      )
+
+      :ok
+    end
+
+    test "returns name and visitor count", %{site: site} do
       mock_http_with("google_analytics_stats.json")
 
-      insert(:google_auth,
-        user: user,
-        site: site,
-        property: "sc-domain:dummy.test",
-        expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
-      )
-
       query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
 
       assert {:ok,
               [
-                %{name: ["keyword1", "keyword2"], visitors: 25},
+                %{name: "keyword1", visitors: 25, ctr: 36.8, impressions: 50, position: 2.2},
-                %{name: ["keyword3", "keyword4"], visitors: 15}
+                %{name: "keyword3", visitors: 15}
               ]} = Google.API.fetch_stats(site, query, 5)
     end
 
-    test "returns next page when page argument is set", %{user: user, site: site} do
+    test "transforms page filters to search console format", %{site: site} do
-      mock_http_with("google_analytics_stats#with_page.json")
+      expect(
-
+        Plausible.HTTPClient.Mock,
-      insert(:google_auth,
+        :post,
-        user: user,
+        fn
-        site: site,
+          "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",
-        property: "sc-domain:dummy.test",
+          [{"Authorization", "Bearer 123"}],
-        expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
+          %{
+            dimensionFilterGroups: [
+              %{filters: [%{expression: "https://dummy.test/page", dimension: "page"}]}
+            ],
+            dimensions: ["query"],
+            endDate: "2022-01-05",
+            rowLimit: 5,
+            startDate: "2022-01-01"
+          } ->
+            {:ok, %Finch.Response{status: 200, body: %{"rows" => []}}}
+        end
       )
 
-      query = %Plausible.Stats.Query{
+      query =
-        filters: %{"page" => 5},
+        Plausible.Stats.Query.from(site, %{
-        date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])
+          "period" => "custom",
-      }
+          "from" => "2022-01-01",
+          "to" => "2022-01-05",
+          "filters" => "event:page==/page"
+        })
 
-      assert {:ok,
+      assert {:ok, []} = Google.API.fetch_stats(site, query, 5)
-              [
-                %{name: ["keyword1", "keyword2"], visitors: 25},
-                %{name: ["keyword3", "keyword4"], visitors: 15}
-              ]} = Google.API.fetch_stats(site, query, 5)
     end
 
-    test "defaults first page when page argument is not set", %{user: user, site: site} do
+    test "returns :invalid filters when using filters that cannot be used in Search Console", %{
-      mock_http_with("google_analytics_stats#without_page.json")
+      site: site
+    } do
+      query =
+        Plausible.Stats.Query.from(site, %{
+          "period" => "custom",
+          "from" => "2022-01-01",
+          "to" => "2022-01-05",
+          "filters" => "event:goal==Signup"
+        })
 
-      insert(:google_auth,
+      assert {:error, :unsupported_filters} = Google.API.fetch_stats(site, query, 5)
-        user: user,
-        site: site,
-        property: "sc-domain:dummy.test",
-        expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
-      )
-
-      query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
-
-      assert {:ok,
-              [
-                %{name: ["keyword1", "keyword2"], visitors: 25},
-                %{name: ["keyword3", "keyword4"], visitors: 15}
-              ]} = Google.API.fetch_stats(site, query, 5)
-    end
-
-    test "returns error when token refresh fails", %{user: user, site: site} do
-      mock_http_with("google_analytics_auth#invalid_grant.json")
-
-      insert(:google_auth,
-        user: user,
-        site: site,
-        property: "sc-domain:dummy.test",
-        access_token: "*****",
-        refresh_token: "*****",
-        expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600)
-      )
-
-      query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
-
-      assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)
     end
   end
 

test/plausible/google/search_console/filters_test.exs

@@ -0,0 +1,183 @@
+defmodule Plausible.Google.SearchConsole.FiltersTest do
+  alias Plausible.Google.SearchConsole.Filters
+  use Plausible.DataCase, async: true
+
+  test "transforms simple page filter" do
+    filters = %{
+      "visit:entry_page" => {:is, "/page"}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{filters: [%{dimension: "page", expression: "https://plausible.io/page"}]}
+           ]
+  end
+
+  test "transforms matches page filter" do
+    filters = %{
+      "visit:entry_page" => {:matches, "*page*"}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{
+                   dimension: "page",
+                   operator: "includingRegex",
+                   expression: "^https://plausible\\.io.*page.*$"
+                 }
+               ]
+             }
+           ]
+  end
+
+  test "transforms member page filter" do
+    filters = %{
+      "visit:entry_page" => {:member, ["/pageA", "/pageB"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{
+                   dimension: "page",
+                   operator: "includingRegex",
+                   expression: "https://plausible.io/pageA|https://plausible.io/pageB"
+                 }
+               ]
+             }
+           ]
+  end
+
+  test "transforms matches_member page filter" do
+    filters = %{
+      "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{
+                   dimension: "page",
+                   operator: "includingRegex",
+                   expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"
+                 }
+               ]
+             }
+           ]
+  end
+
+  test "transforms event:page exactly like visit:entry_page" do
+    filters = %{
+      "event:page" => {:matches_member, ["/pageA*", "/pageB*"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{
+                   dimension: "page",
+                   operator: "includingRegex",
+                   expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"
+                 }
+               ]
+             }
+           ]
+  end
+
+  test "transforms simple visit:screen filter" do
+    filters = %{
+      "visit:screen" => {:is, "Desktop"}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [%{filters: [%{dimension: "device", expression: "DESKTOP"}]}]
+  end
+
+  test "transforms member visit:screen filter" do
+    filters = %{
+      "visit:screen" => {:member, ["Mobile", "Tablet"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{dimension: "device", operator: "includingRegex", expression: "Mobile|Tablet"}
+               ]
+             }
+           ]
+  end
+
+  test "transforms simple visit:country filter to alpha3" do
+    filters = %{
+      "visit:country" => {:is, "EE"}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [%{filters: [%{dimension: "country", expression: "EST"}]}]
+  end
+
+  test "transforms member visit:country filter" do
+    filters = %{
+      "visit:country" => {:member, ["EE", "PL"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{dimension: "country", operator: "includingRegex", expression: "EST|POL"}
+               ]
+             }
+           ]
+  end
+
+  test "filters can be combined" do
+    filters = %{
+      "visit:entry_page" => {:matches, "*web-analytics*"},
+      "visit:screen" => {:is, "Desktop"},
+      "visit:country" => {:member, ["EE", "PL"]}
+    }
+
+    {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
+
+    assert transformed == [
+             %{
+               filters: [
+                 %{dimension: "device", expression: "DESKTOP"},
+                 %{
+                   dimension: "page",
+                   operator: "includingRegex",
+                   expression: "^https://plausible\\.io.*web\\-analytics.*$"
+                 },
+                 %{dimension: "country", operator: "includingRegex", expression: "EST|POL"}
+               ]
+             }
+           ]
+  end
+
+  test "when unsupported filter is included the whole set becomes invalid" do
+    filters = %{
+      "visit:entry_page" => {:matches, "*web-analytics*"},
+      "visit:screen" => {:is, "Desktop"},
+      "visit:country" => {:member, ["EE", "PL"]},
+      "visit:utm_medium" => {:is, "facebook"}
+    }
+
+    assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters)
+  end
+end

test/plausible_web/controllers/api/stats_controller/sources_test.exs

@@ -1606,9 +1606,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
              ]
     end
 
-    test "gets keywords from Google", %{conn: conn, user: user, site: site} do
+    test "gets keywords from Google", %{conn: conn, site: site} do
-      insert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com")
-
       populate_stats(site, [
         build(:pageview,
           referrer_source: "DuckDuckGo",
@@ -1627,10 +1625,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
       conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day")
       {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil)
 
-      assert json_response(conn, 200) == %{
+      assert json_response(conn, 200) == %{"search_terms" => terms}
-               "total_visitors" => 2,
-               "search_terms" => terms
-             }
     end
 
     test "works when filter expression is provided for source", %{

test/test_helper.exs

@@ -23,7 +23,7 @@ if :minio in Keyword.fetch!(ExUnit.configuration(), :include) do
 end
 
 if Mix.env() == :ce_test do
-  IO.puts("Test mode: Communnity Edition")
+  IO.puts("Test mode: Community Edition")
   ExUnit.configure(exclude: [:slow, :minio, :ee_only])
 else
   IO.puts("Test mode: Enterprise Edition")