plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-12-23 17:44:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refresh Google Analytics token before import (#2254)

Browse Source 
* Capture refresh and expires from GA callback

* Pass GA refresh token to import worker

* Refresh GA token before import
This commit is contained in:
Vinicius Brasil 2022-09-26 06:29:56 -03:00 committed by GitHub
parent 0f7f82f789
commit 1adda42a75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 152 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
lib/plausible/google/api.ex
View File

@ -30,8 +30,8 @@ defmodule Plausible.Google.Api do
end
def fetch_verified_properties(auth) do
with {:ok, auth} <- refresh_if_needed(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do
with {:ok, access_token} <- maybe_refresh_token(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do
sites
|> Map.get("siteEntry", [])
|> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
@ -43,10 +43,15 @@ defmodule Plausible.Google.Api do
def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
with site <- Plausible.Repo.preload(site, :google_auth),
{:ok, %{access_token: access_token, property: property}} <-
refresh_if_needed(site.google_auth),
{:ok, access_token} <- maybe_refresh_token(site.google_auth),
{:ok, stats} <-
HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do
HTTP.list_stats(
access_token,
site.google_auth.property,
date_range,
limit,
filters["page"]
) do
stats
|> Map.get("rows", [])
|> Enum.filter(fn row -> row["clicks"] > 0 end)
@ -103,9 +108,15 @@ defmodule Plausible.Google.Api do
end
end
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 10_000
@max_attempts 5
@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) ::
@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
@ -125,7 +136,21 @@ defmodule Plausible.Google.Api do
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(site, date_range, view_id, access_token) do
def import_analytics(site, date_range, view_id, auth) do
with {:ok, access_token} <- maybe_refresh_token(auth),
:ok <- do_import_analytics(site, date_range, view_id, access_token) do
:ok
else
{:error, cause} ->
Sentry.capture_message("Failed to import from Google Analytics",
extra: %{site: site.domain, error: inspect(cause)}
)
{:error, cause}
end
end
defp do_import_analytics(site, date_range, view_id, access_token) do
{:ok, buffer} = Plausible.Google.Buffer.start_link()
result =
@ -176,10 +201,6 @@ defmodule Plausible.Google.Api do
{:error, cause} ->
if attempt >= @max_attempts do
Sentry.capture_message("Failed to import from Google Analytics",
extra: %{site: site.domain, error: inspect(cause)}
)
{:error, cause}
else
Process.sleep(sleep_time)
@ -188,28 +209,56 @@ defmodule Plausible.Google.Api do
end
end
defp refresh_if_needed(auth) do
if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do
do_refresh_token(auth)
defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
with true <- needs_to_refresh_token?(auth.expires),
{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
changeset <-
Plausible.Site.GoogleAuth.changeset(auth, %{
access_token: new_access_token,
expires: expires_at
}),
{:ok, _google_auth} <- Plausible.Repo.update(changeset) do
{:ok, new_access_token}
else
{:ok, auth}
false -> {:ok, auth.access_token}
{:error, cause} -> {:error, cause}
end
end
defp do_refresh_token(auth) do
case HTTP.refresh_auth_token(auth.refresh_token) do
{:ok, %{"access_token" => access_token, "expires_in" => expires_in}} ->
expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
defp maybe_refresh_token({access_token, nil, nil}) do
{:ok, access_token}
end
auth
|> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in})
|> Plausible.Repo.update()
error ->
error
defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
if needs_to_refresh_token?(expires_at) do
do_refresh_token(refresh_token)
else
{:ok, access_token}
end
end
defp do_refresh_token(refresh_token) do
case HTTP.refresh_auth_token(refresh_token) do
{:ok, %{"access_token" => new_access_token, "expires_at" => expires_in}} ->
expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
{:ok, {new_access_token, expires_at}}
{:error, cause} ->
{:error, cause}
end
end
defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do
expires_at
|> NaiveDateTime.from_iso8601!()
|> needs_to_refresh_token?()
end
defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do
thirty_seconds_ago = Timex.shift(Timex.now(), seconds: 30)
Timex.before?(expires_at, thirty_seconds_ago)
end
defp client_id() do
Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
end

7
lib/plausible_web/controllers/auth_controller.ex
View File

@ -546,13 +546,16 @@ defmodule PlausibleWeb.AuthController do
res = Plausible.Google.HTTP.fetch_access_token(code)
[site_id, redirect_to] = Jason.decode!(state)
site = Repo.get(Plausible.Site, site_id)
expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])
case redirect_to do
"import" ->
redirect(conn,
to:
Routes.site_path(conn, :import_from_google_view_id_form, site.domain,
access_token: res["access_token"]
access_token: res["access_token"],
refresh_token: res["refresh_token"],
expires_at: NaiveDateTime.to_iso8601(expires_at)
)
)
@ -565,7 +568,7 @@ defmodule PlausibleWeb.AuthController do
email: id["email"],
refresh_token: res["refresh_token"],
access_token: res["access_token"],
expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),
expires_at: expires_at,
user_id: conn.assigns[:current_user].id,
site_id: site_id
})

48
lib/plausible_web/controllers/site_controller.ex
View File

@ -636,7 +636,9 @@ defmodule PlausibleWeb.SiteController do
def import_from_google_user_metric_notice(conn, %{
"view_id" => view_id,
"access_token" => access_token
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns[:site]
@ -646,11 +648,17 @@ defmodule PlausibleWeb.SiteController do
site: site,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def import_from_google_view_id_form(conn, %{"access_token" => access_token}) do
def import_from_google_view_id_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns[:site]
view_ids = Plausible.Google.Api.list_views(access_token)
@ -658,6 +666,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
view_ids: view_ids,
layout: {PlausibleWeb.LayoutView, "focus.html"}
@ -666,7 +676,12 @@ defmodule PlausibleWeb.SiteController do
# see https://stackoverflow.com/a/57416769
@google_analytics_new_user_metric_date ~D[2016-08-24]
def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do
def import_from_google_view_id(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@ -679,6 +694,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
view_ids: view_ids,
selected_view_id_error: "No data found. Nothing to import",
@ -691,7 +708,9 @@ defmodule PlausibleWeb.SiteController do
to:
Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,
view_id: view_id,
access_token: access_token
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at
)
)
else
@ -699,14 +718,21 @@ defmodule PlausibleWeb.SiteController do
to:
Routes.site_path(conn, :import_from_google_confirm, site.domain,
view_id: view_id,
access_token: access_token
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at
)
)
end
end
end
def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do
def import_from_google_confirm(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@ -720,6 +746,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_view_id: view_id,
selected_view_id_name: view_name,
@ -733,7 +761,9 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns[:site]
@ -743,7 +773,9 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token
"access_token" => access_token,
"refresh_token" => refresh_token,
"token_expires_at" => expires_at
})
Ecto.Multi.new()

2
lib/plausible_web/templates/site/import_from_google_confirm.html.eex
View File

@ -2,6 +2,8 @@
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= case @start_date do %>
<% {:ok, start_date} -> %>

2
lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
View File

@ -22,5 +22,5 @@
</p>
</div>
<%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token), class: "button mt-6") %>
<%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %>
</div>

2
lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
View File

@ -2,6 +2,8 @@
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= case @view_ids do %>
<% {:ok, view_ids} -> %>

18
lib/workers/import_google_analytics.ex
View File

@ -9,13 +9,13 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
@impl Oban.Worker
def perform(
%Oban.Job{
args: %{
"site_id" => site_id,
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token
}
args:
%{
"site_id" => site_id,
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date
} = args
},
google_api \\ Plausible.Google.Api
) do
@ -24,7 +24,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
end_date = Date.from_iso8601!(end_date)
date_range = Date.range(start_date, end_date)
case google_api.import_analytics(site, date_range, view_id, access_token) do
auth = {args["access_token"], args["refresh_token"], args["token_expires_at"]}
case google_api.import_analytics(site, date_range, view_id, auth) do
:ok ->
Plausible.Site.import_success(site)
|> Repo.update!()

6
test/plausible/google/vcr_test.exs
View File

@ -21,11 +21,13 @@ defmodule Plausible.Google.Api.VCRTest do
inserts_before_importing = get_insert_count()
before_importing_timestamp = DateTime.utc_now()
access_token = "***"
view_id = "54297898"
date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token)
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()
auth = {"***", "refresh_token", future}
assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, auth)
total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second)
total_inserts = get_insert_count() - inserts_before_importing

18
test/plausible_web/controllers/site_controller_test.exs
View File

@ -745,7 +745,11 @@ defmodule PlausibleWeb.SiteControllerTest do
response =
conn
|> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token"})
|> get("/#{site.domain}/import/google-analytics/view-id", %{
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777"
})
|> html_response(200)
assert response =~ "57238190 - one.test"
@ -761,7 +765,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
"access_token" => "token"
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777"
})
imported_data = Repo.reload(site).imported_data
@ -777,7 +783,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
"access_token" => "token"
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777"
})
assert_enqueued(
@ -787,7 +795,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
"access_token" => "token"
"access_token" => "token",
"refresh_token" => "foo",
"token_expires_at" => "2022-09-22T20:01:37.112777"
}
)
end