Refresh Google Analytics token before import (#2254)

* Capture refresh and expires from GA callback

* Pass GA refresh token to import worker

* Refresh GA token before import

lib/plausible/google/api.ex

@@ -30,8 +30,8 @@ defmodule Plausible.Google.Api do
   end
 
   def fetch_verified_properties(auth) do
-    with {:ok, auth} <- refresh_if_needed(auth),
+    with {:ok, access_token} <- maybe_refresh_token(auth),
-         {:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do
+         {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do
       sites
       |> Map.get("siteEntry", [])
       |> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
@@ -43,10 +43,15 @@ defmodule Plausible.Google.Api do
 
   def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
     with site <- Plausible.Repo.preload(site, :google_auth),
-         {:ok, %{access_token: access_token, property: property}} <-
+         {:ok, access_token} <- maybe_refresh_token(site.google_auth),
-           refresh_if_needed(site.google_auth),
          {:ok, stats} <-
-           HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do
+           HTTP.list_stats(
+             access_token,
+             site.google_auth.property,
+             date_range,
+             limit,
+             filters["page"]
+           ) do
       stats
       |> Map.get("rows", [])
       |> Enum.filter(fn row -> row["clicks"] > 0 end)
@@ -103,9 +108,15 @@ defmodule Plausible.Google.Api do
     end
   end
 
+  @type import_auth :: {
+          access_token :: String.t(),
+          refresh_token :: String.t(),
+          expires_at :: String.t()
+        }
+
   @per_page 10_000
   @max_attempts 5
-  @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) ::
+  @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
           :ok | {:error, term()}
   @doc """
   Imports stats from a Google Analytics UA view to a Plausible site.
@@ -125,7 +136,21 @@ defmodule Plausible.Google.Api do
   - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
 
   """
-  def import_analytics(site, date_range, view_id, access_token) do
+  def import_analytics(site, date_range, view_id, auth) do
+    with {:ok, access_token} <- maybe_refresh_token(auth),
+         :ok <- do_import_analytics(site, date_range, view_id, access_token) do
+      :ok
+    else
+      {:error, cause} ->
+        Sentry.capture_message("Failed to import from Google Analytics",
+          extra: %{site: site.domain, error: inspect(cause)}
+        )
+
+        {:error, cause}
+    end
+  end
+
+  defp do_import_analytics(site, date_range, view_id, access_token) do
     {:ok, buffer} = Plausible.Google.Buffer.start_link()
 
     result =
@@ -176,10 +201,6 @@ defmodule Plausible.Google.Api do
 
       {:error, cause} ->
         if attempt >= @max_attempts do
-          Sentry.capture_message("Failed to import from Google Analytics",
-            extra: %{site: site.domain, error: inspect(cause)}
-          )
-
           {:error, cause}
         else
           Process.sleep(sleep_time)
@@ -188,28 +209,56 @@ defmodule Plausible.Google.Api do
     end
   end
 
-  defp refresh_if_needed(auth) do
+  defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
-    if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do
+    with true <- needs_to_refresh_token?(auth.expires),
-      do_refresh_token(auth)
+         {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
+         changeset <-
+           Plausible.Site.GoogleAuth.changeset(auth, %{
+             access_token: new_access_token,
+             expires: expires_at
+           }),
+         {:ok, _google_auth} <- Plausible.Repo.update(changeset) do
+      {:ok, new_access_token}
     else
-      {:ok, auth}
+      false -> {:ok, auth.access_token}
+      {:error, cause} -> {:error, cause}
     end
   end
 
-  defp do_refresh_token(auth) do
+  defp maybe_refresh_token({access_token, nil, nil}) do
-    case HTTP.refresh_auth_token(auth.refresh_token) do
+    {:ok, access_token}
-      {:ok, %{"access_token" => access_token, "expires_in" => expires_in}} ->
+  end
-        expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
 
-        auth
+  defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
-        |> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in})
+    if needs_to_refresh_token?(expires_at) do
-        |> Plausible.Repo.update()
+      do_refresh_token(refresh_token)
-
+    else
-      error ->
+      {:ok, access_token}
-        error
     end
   end
 
+  defp do_refresh_token(refresh_token) do
+    case HTTP.refresh_auth_token(refresh_token) do
+      {:ok, %{"access_token" => new_access_token, "expires_at" => expires_in}} ->
+        expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
+        {:ok, {new_access_token, expires_at}}
+
+      {:error, cause} ->
+        {:error, cause}
+    end
+  end
+
+  defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do
+    expires_at
+    |> NaiveDateTime.from_iso8601!()
+    |> needs_to_refresh_token?()
+  end
+
+  defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do
+    thirty_seconds_ago = Timex.shift(Timex.now(), seconds: 30)
+    Timex.before?(expires_at, thirty_seconds_ago)
+  end
+
   defp client_id() do
     Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
   end

lib/plausible_web/controllers/auth_controller.ex

@@ -546,13 +546,16 @@ defmodule PlausibleWeb.AuthController do
     res = Plausible.Google.HTTP.fetch_access_token(code)
     [site_id, redirect_to] = Jason.decode!(state)
     site = Repo.get(Plausible.Site, site_id)
+    expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])
 
     case redirect_to do
       "import" ->
         redirect(conn,
           to:
             Routes.site_path(conn, :import_from_google_view_id_form, site.domain,
-              access_token: res["access_token"]
+              access_token: res["access_token"],
+              refresh_token: res["refresh_token"],
+              expires_at: NaiveDateTime.to_iso8601(expires_at)
             )
         )
 
@@ -565,7 +568,7 @@ defmodule PlausibleWeb.AuthController do
           email: id["email"],
           refresh_token: res["refresh_token"],
           access_token: res["access_token"],
-          expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),
+          expires_at: expires_at,
           user_id: conn.assigns[:current_user].id,
           site_id: site_id
         })

lib/plausible_web/controllers/site_controller.ex

@@ -636,7 +636,9 @@ defmodule PlausibleWeb.SiteController do
 
   def import_from_google_user_metric_notice(conn, %{
         "view_id" => view_id,
-        "access_token" => access_token
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "expires_at" => expires_at
       }) do
     site = conn.assigns[:site]
 
@@ -646,11 +648,17 @@ defmodule PlausibleWeb.SiteController do
       site: site,
       view_id: view_id,
       access_token: access_token,
+      refresh_token: refresh_token,
+      expires_at: expires_at,
       layout: {PlausibleWeb.LayoutView, "focus.html"}
     )
   end
 
-  def import_from_google_view_id_form(conn, %{"access_token" => access_token}) do
+  def import_from_google_view_id_form(conn, %{
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "expires_at" => expires_at
+      }) do
     site = conn.assigns[:site]
     view_ids = Plausible.Google.Api.list_views(access_token)
 
@@ -658,6 +666,8 @@ defmodule PlausibleWeb.SiteController do
     |> assign(:skip_plausible_tracking, true)
     |> render("import_from_google_view_id_form.html",
       access_token: access_token,
+      refresh_token: refresh_token,
+      expires_at: expires_at,
       site: site,
       view_ids: view_ids,
       layout: {PlausibleWeb.LayoutView, "focus.html"}
@@ -666,7 +676,12 @@ defmodule PlausibleWeb.SiteController do
 
   # see https://stackoverflow.com/a/57416769
   @google_analytics_new_user_metric_date ~D[2016-08-24]
-  def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do
+  def import_from_google_view_id(conn, %{
+        "view_id" => view_id,
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "expires_at" => expires_at
+      }) do
     site = conn.assigns[:site]
     start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
 
@@ -679,6 +694,8 @@ defmodule PlausibleWeb.SiteController do
         |> assign(:skip_plausible_tracking, true)
         |> render("import_from_google_view_id_form.html",
           access_token: access_token,
+          refresh_token: refresh_token,
+          expires_at: expires_at,
           site: site,
           view_ids: view_ids,
           selected_view_id_error: "No data found. Nothing to import",
@@ -691,7 +708,9 @@ defmodule PlausibleWeb.SiteController do
             to:
               Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,
                 view_id: view_id,
-                access_token: access_token
+                access_token: access_token,
+                refresh_token: refresh_token,
+                expires_at: expires_at
               )
           )
         else
@@ -699,14 +718,21 @@ defmodule PlausibleWeb.SiteController do
             to:
               Routes.site_path(conn, :import_from_google_confirm, site.domain,
                 view_id: view_id,
-                access_token: access_token
+                access_token: access_token,
+                refresh_token: refresh_token,
+                expires_at: expires_at
               )
           )
         end
     end
   end
 
-  def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do
+  def import_from_google_confirm(conn, %{
+        "view_id" => view_id,
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "expires_at" => expires_at
+      }) do
     site = conn.assigns[:site]
 
     start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@@ -720,6 +746,8 @@ defmodule PlausibleWeb.SiteController do
     |> assign(:skip_plausible_tracking, true)
     |> render("import_from_google_confirm.html",
       access_token: access_token,
+      refresh_token: refresh_token,
+      expires_at: expires_at,
       site: site,
       selected_view_id: view_id,
       selected_view_id_name: view_name,
@@ -733,7 +761,9 @@ defmodule PlausibleWeb.SiteController do
         "view_id" => view_id,
         "start_date" => start_date,
         "end_date" => end_date,
-        "access_token" => access_token
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "expires_at" => expires_at
       }) do
     site = conn.assigns[:site]
 
@@ -743,7 +773,9 @@ defmodule PlausibleWeb.SiteController do
         "view_id" => view_id,
         "start_date" => start_date,
         "end_date" => end_date,
-        "access_token" => access_token
+        "access_token" => access_token,
+        "refresh_token" => refresh_token,
+        "token_expires_at" => expires_at
       })
 
     Ecto.Multi.new()

lib/plausible_web/templates/site/import_from_google_confirm.html.eex

@@ -2,6 +2,8 @@
   <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
 
   <%= hidden_input(f, :access_token, value: @access_token) %>
+  <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+  <%= hidden_input(f, :expires_at, value: @expires_at) %>
 
   <%= case @start_date do %>
     <% {:ok, start_date} -> %>

lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex

@@ -22,5 +22,5 @@
     </p>
   </div>
 
-  <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token), class: "button mt-6") %>
+  <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %>
 </div>

lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex

@@ -2,6 +2,8 @@
   <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
 
   <%= hidden_input(f, :access_token, value: @access_token) %>
+  <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+  <%= hidden_input(f, :expires_at, value: @expires_at) %>
 
   <%= case @view_ids do %>
     <% {:ok, view_ids} -> %>

lib/workers/import_google_analytics.ex

@@ -9,13 +9,13 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
   @impl Oban.Worker
   def perform(
         %Oban.Job{
-          args: %{
+          args:
-            "site_id" => site_id,
+            %{
-            "view_id" => view_id,
+              "site_id" => site_id,
-            "start_date" => start_date,
+              "view_id" => view_id,
-            "end_date" => end_date,
+              "start_date" => start_date,
-            "access_token" => access_token
+              "end_date" => end_date
-          }
+            } = args
         },
         google_api \\ Plausible.Google.Api
       ) do
@@ -24,7 +24,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
     end_date = Date.from_iso8601!(end_date)
     date_range = Date.range(start_date, end_date)
 
-    case google_api.import_analytics(site, date_range, view_id, access_token) do
+    auth = {args["access_token"], args["refresh_token"], args["token_expires_at"]}
+
+    case google_api.import_analytics(site, date_range, view_id, auth) do
       :ok ->
         Plausible.Site.import_success(site)
         |> Repo.update!()

test/plausible/google/vcr_test.exs

@@ -21,11 +21,13 @@ defmodule Plausible.Google.Api.VCRTest do
       inserts_before_importing = get_insert_count()
       before_importing_timestamp = DateTime.utc_now()
 
-      access_token = "***"
       view_id = "54297898"
       date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
 
-      assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token)
+      future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()
+      auth = {"***", "refresh_token", future}
+
+      assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, auth)
 
       total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second)
       total_inserts = get_insert_count() - inserts_before_importing

test/plausible_web/controllers/site_controller_test.exs

@@ -745,7 +745,11 @@ defmodule PlausibleWeb.SiteControllerTest do
 
       response =
         conn
-        |> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token"})
+        |> get("/#{site.domain}/import/google-analytics/view-id", %{
+          "access_token" => "token",
+          "refresh_token" => "foo",
+          "expires_at" => "2022-09-22T20:01:37.112777"
+        })
         |> html_response(200)
 
       assert response =~ "57238190 - one.test"
@@ -761,7 +765,9 @@ defmodule PlausibleWeb.SiteControllerTest do
         "view_id" => "123",
         "start_date" => "2018-03-01",
         "end_date" => "2022-03-01",
-        "access_token" => "token"
+        "access_token" => "token",
+        "refresh_token" => "foo",
+        "expires_at" => "2022-09-22T20:01:37.112777"
       })
 
       imported_data = Repo.reload(site).imported_data
@@ -777,7 +783,9 @@ defmodule PlausibleWeb.SiteControllerTest do
         "view_id" => "123",
         "start_date" => "2018-03-01",
         "end_date" => "2022-03-01",
-        "access_token" => "token"
+        "access_token" => "token",
+        "refresh_token" => "foo",
+        "expires_at" => "2022-09-22T20:01:37.112777"
       })
 
       assert_enqueued(
@@ -787,7 +795,9 @@ defmodule PlausibleWeb.SiteControllerTest do
           "view_id" => "123",
           "start_date" => "2018-03-01",
           "end_date" => "2022-03-01",
-          "access_token" => "token"
+          "access_token" => "token",
+          "refresh_token" => "foo",
+          "token_expires_at" => "2022-09-22T20:01:37.112777"
         }
       )
     end