From 203f87520b04d9df7653aff3c224db684d26c98b Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 22 Mar 2022 16:09:45 +0200 Subject: [PATCH] Ga import improvements (#1784) * Do not link Google account for import * Record start date * Fix tests --- assets/js/dashboard/datepicker.js | 6 +- assets/js/dashboard/mount.js | 2 +- lib/mix/tasks/call_google.ex | 48 ------ lib/plausible/google/api.ex | 149 +++++++++--------- lib/plausible/site/schema.ex | 8 +- lib/plausible/sites.ex | 8 + lib/plausible/stats/clickhouse.ex | 18 +++ .../controllers/auth_controller.ex | 44 ++++-- .../controllers/site_controller.ex | 100 +++++++----- .../controllers/stats_controller.ex | 2 + lib/plausible_web/router.ex | 2 + .../site/import_from_google.html.eex | 21 +-- .../site/import_from_google_confirm.html.eex | 38 +++++ .../templates/site/settings_general.html.eex | 63 ++------ .../templates/stats/stats.html.eex | 2 +- lib/workers/import_google_analytics.ex | 16 +- .../controllers/site_controller_test.exs | 18 ++- test/support/test_utils.ex | 2 +- test/workers/import_google_analytics_test.exs | 56 ++++++- 19 files changed, 338 insertions(+), 265 deletions(-) delete mode 100644 lib/mix/tasks/call_google.ex create mode 100644 lib/plausible_web/templates/site/import_from_google_confirm.html.eex diff --git a/assets/js/dashboard/datepicker.js b/assets/js/dashboard/datepicker.js index d83f88667..0f3d6ad19 100644 --- a/assets/js/dashboard/datepicker.js +++ b/assets/js/dashboard/datepicker.js @@ -22,7 +22,7 @@ import { import { navigateToQuery, QueryLink, QueryButton } from "./query"; function renderArrow(query, site, period, prevDate, nextDate) { - const insertionDate = parseUTCDate(site.insertedAt); + const insertionDate = parseUTCDate(site.statsBegin); const disabledLeft = isBefore( parseUTCDate(prevDate), insertionDate, @@ -136,7 +136,7 @@ class DatePicker extends React.Component { date: false, }; - const insertionDate = parseUTCDate(this.props.site.insertedAt); + const insertionDate = parseUTCDate(this.props.site.statsBegin); if (e.key === "ArrowLeft") { const prevDate = formatISO(shiftDays(query.date, -1)); @@ -358,7 +358,7 @@ class DatePicker extends React.Component { ); } if (this.state.mode === "calendar") { - const insertionDate = new Date(this.props.site.insertedAt); + const insertionDate = new Date(this.props.site.statsBegin); const dayBeforeCreation = insertionDate - 86400000; return (
diff --git a/assets/js/dashboard/mount.js b/assets/js/dashboard/mount.js index 635d2e5c4..38cc914a3 100644 --- a/assets/js/dashboard/mount.js +++ b/assets/js/dashboard/mount.js @@ -13,7 +13,7 @@ if (container) { domain: container.dataset.domain, offset: container.dataset.offset, hasGoals: container.dataset.hasGoals === 'true', - insertedAt: container.dataset.insertedAt, + statsBegin: container.dataset.statsBegin, embedded: container.dataset.embedded, background: container.dataset.background, selfhosted: container.dataset.selfhosted === 'true' diff --git a/lib/mix/tasks/call_google.ex b/lib/mix/tasks/call_google.ex deleted file mode 100644 index 32fdad203..000000000 --- a/lib/mix/tasks/call_google.ex +++ /dev/null @@ -1,48 +0,0 @@ -defmodule Mix.Tasks.CallGoogle do - use Mix.Task - use Plausible.Repo - require Logger - - @token "ya29.A0ARrdaM-GC-C38oUNmoUlica3v2IGT1eACtvreAFcM4zCYrpQxHdTFfhpmWQfiYAblnKPVRkt_fW-jTTwJ_ScXtkaqq6P_XiBbPIa3g9yJ_F6bP2tEOeHgq07aPlPwHvqXaK8I8gmwtO86VvJLvmhXolHtlXN0g" - @view_id "204615196" - @dimensions ["ga:date"] - @metrics ["ga:users", "ga:pageviews", "ga:bounces", "ga:sessions", "ga:sessionDuration"] - - def run(_) do - Mix.Task.run("app.start") - - report = %{ - viewId: @view_id, - dateRanges: [ - %{ - # The earliest valid date - startDate: "2005-01-01", - endDate: "2019-12-05" - } - ], - dimensions: Enum.map(@dimensions, &%{name: &1, histogramBuckets: []}), - metrics: Enum.map(@metrics, &%{expression: &1}), - hideTotals: true, - hideValueRanges: true, - orderBys: [ - %{ - fieldName: "ga:date", - sortOrder: "DESCENDING" - } - ], - pageSize: 100_00, - pageToken: "" - } - - Logger.debug(report) - - res = - HTTPoison.post!( - "https://analyticsreporting.googleapis.com/v4/reports:batchGet", - Jason.encode!(%{reportRequests: [report]}), - Authorization: "Bearer #{@token}" - ) - - Logger.debug(res.body) - end -end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 942ed1b30..91f9c28ad 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -6,6 +6,7 @@ defmodule Plausible.Google.Api do @scope URI.encode_www_form( "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly" ) + @import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly") @verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"] def authorize_url(site_id, redirect_to) do @@ -17,6 +18,15 @@ defmodule Plausible.Google.Api do end end + def import_authorize_url(site_id, redirect_to) do + if Application.get_env(:plausible, :environment) == "test" do + "" + else + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> + Jason.encode!([site_id, redirect_to]) + end + end + def fetch_access_token(code) do res = HTTPoison.post!( @@ -120,17 +130,11 @@ defmodule Plausible.Google.Api do end end - def get_analytics_view_ids(site) do - with {:ok, auth} <- refresh_if_needed(site.google_auth) do - do_get_analytics_view_ids(auth) - end - end - - def do_get_analytics_view_ids(auth) do + def get_analytics_view_ids(token) do res = HTTPoison.get!( "https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles", - Authorization: "Bearer #{auth.access_token}" + Authorization: "Bearer #{token}" ) case res.status_code do @@ -153,11 +157,60 @@ defmodule Plausible.Google.Api do end end - def import_analytics(site, view_id, end_date) do - with {:ok, auth} <- refresh_if_needed(site.google_auth) do - do_import_analytics(site, auth, view_id, end_date) - end - end + # Each element is: {dataset, dimensions, metrics} + @request_data [ + { + "imported_visitors", + ["ga:date"], + [ + "ga:users", + "ga:pageviews", + "ga:bounces", + "ga:sessions", + "ga:sessionDuration" + ] + }, + { + "imported_sources", + ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, + { + "imported_pages", + ["ga:date", "ga:hostname", "ga:pagePath"], + ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"] + }, + { + "imported_entry_pages", + ["ga:date", "ga:landingPagePath"], + ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] + }, + { + "imported_exit_pages", + ["ga:date", "ga:exitPagePath"], + ["ga:users", "ga:exits"] + }, + { + "imported_locations", + ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, + { + "imported_devices", + ["ga:date", "ga:deviceCategory"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, + { + "imported_browsers", + ["ga:date", "ga:browser"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, + { + "imported_operating_systems", + ["ga:date", "ga:operatingSystem"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + } + ] @doc """ API reference: @@ -165,71 +218,17 @@ defmodule Plausible.Google.Api do Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer """ - def do_import_analytics(site, auth, view_id, end_date) do + def import_analytics(site, view_id, start_date, end_date, access_token) do request = %{ - auth: auth, - profile: view_id, + access_token: access_token, + view_id: view_id, + start_date: start_date, end_date: end_date } - # Each element is: {dataset, dimensions, metrics} - request_data = [ - { - "imported_visitors", - ["ga:date"], - [ - "ga:users", - "ga:pageviews", - "ga:bounces", - "ga:sessions", - "ga:sessionDuration" - ] - }, - { - "imported_sources", - ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_pages", - ["ga:date", "ga:hostname", "ga:pagePath"], - ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"] - }, - { - "imported_entry_pages", - ["ga:date", "ga:landingPagePath"], - ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] - }, - { - "imported_exit_pages", - ["ga:date", "ga:exitPagePath"], - ["ga:users", "ga:exits"] - }, - { - "imported_locations", - ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_devices", - ["ga:date", "ga:deviceCategory"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_browsers", - ["ga:date", "ga:browser"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_operating_systems", - ["ga:date", "ga:operatingSystem"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - } - ] - responses = Enum.map( - request_data, + @request_data, fn {dataset, dimensions, metrics} -> fetch_analytic_reports(dataset, dimensions, metrics, request) end @@ -272,11 +271,11 @@ defmodule Plausible.Google.Api do defp fetch_analytic_reports(dataset, dimensions, metrics, request, page_token \\ "") do report = %{ - viewId: request.profile, + viewId: request.view_id, dateRanges: [ %{ # The earliest valid date - startDate: "2005-01-01", + startDate: request.start_date, endDate: request.end_date } ], @@ -298,7 +297,7 @@ defmodule Plausible.Google.Api do HTTPoison.post!( "https://analyticsreporting.googleapis.com/v4/reports:batchGet", Jason.encode!(%{reportRequests: [report]}), - [Authorization: "Bearer #{request.auth.access_token}"], + [Authorization: "Bearer #{request.access_token}"], timeout: 30_000, recv_timeout: 30_000 ) diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 8dee788c0..127dffc01 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -2,6 +2,7 @@ defmodule Plausible.Site.ImportedData do use Ecto.Schema embedded_schema do + field :start_date, :date field :end_date, :date field :source, :string field :status, :string @@ -62,10 +63,11 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end - def start_import(site, imported_source, status \\ "importing") do + def start_import(site, start_date, end_date, imported_source, status \\ "importing") do change(site, - imported_data: %Plausible.Site.ImportedData{ - end_date: Timex.today(), + imported_data: %{ + start_date: start_date, + end_date: end_date, source: imported_source, status: status } diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 92c783bfc..25bba9c84 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -55,6 +55,14 @@ defmodule Plausible.Sites do end end + def stats_begin(site) do + if site.imported_data do + site.imported_data.start_date + else + site.inserted_at + end + end + def create_shared_link(site, name, password \\ nil) do changes = SharedLink.changeset( diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 545f66ce4..143053aa0 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -5,6 +5,24 @@ defmodule Plausible.Stats.Clickhouse do use Plausible.Stats.Fragments @no_ref "Direct / None" + def pageview_start_date_local(site) do + date = + ClickhouseRepo.one( + from e in "events", + select: fragment("toDate(min(?))", e.timestamp), + where: e.domain == ^site.domain + ) + + case date do + # no stats for this domain yet + ~D[1970-01-01] -> + Timex.today(site.timezone) + + date -> + Timex.Timezone.convert(date, site.timezone) + end + end + def usage_breakdown(domains) do range = Date.range( diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 917e0201d..d1c22ac5a 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -538,24 +538,36 @@ defmodule PlausibleWeb.AuthController do def google_auth_callback(conn, %{"code" => code, "state" => state}) do res = Plausible.Google.Api.fetch_access_token(code) - id_token = res["id_token"] - [_, body, _] = String.split(id_token, ".") - id = body |> Base.decode64!(padding: false) |> Jason.decode!() - [site_id, redirect_to] = Jason.decode!(state) - - Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{ - email: id["email"], - refresh_token: res["refresh_token"], - access_token: res["access_token"], - expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]), - user_id: conn.assigns[:current_user].id, - site_id: site_id - }) - |> Repo.insert!() - site = Repo.get(Plausible.Site, site_id) - redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}") + case redirect_to do + "import" -> + redirect(conn, + to: + Routes.site_path(conn, :import_from_google_form, site.domain, + access_token: res["access_token"] + ) + ) + + _ -> + id_token = res["id_token"] + [_, body, _] = String.split(id_token, ".") + id = body |> Base.decode64!(padding: false) |> Jason.decode!() + + Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{ + email: id["email"], + refresh_token: res["refresh_token"], + access_token: res["access_token"], + expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]), + user_id: conn.assigns[:current_user].id, + site_id: site_id + }) + |> Repo.insert!() + + site = Repo.get(Plausible.Site, site_id) + + redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}") + end end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index c53cabb48..cb9052325 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -167,28 +167,15 @@ defmodule PlausibleWeb.SiteController do redirect(conn, to: Routes.site_path(conn, :settings_general, website)) end - defp can_trigger_import(site) do - no_import = is_nil(site.imported_data) || site.imported_data.status == "error" - - no_import && site.google_auth - end - def settings_general(conn, _params) do site = conn.assigns[:site] |> Repo.preload([:custom_domain, :google_auth]) - google_profiles = - if can_trigger_import(site) do - Plausible.Google.Api.get_analytics_view_ids(site) - end - conn |> assign(:skip_plausible_tracking, true) |> render("settings_general.html", site: site, - google_profiles: google_profiles, - imported_data: site.imported_data, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) @@ -643,48 +630,81 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings/general") end - def import_from_google_form(conn, _params) do - site = conn.assigns[:site] |> Repo.preload(:google_auth) + def import_from_google_form(conn, %{"access_token" => access_token}) do + site = conn.assigns[:site] - view_ids = Plausible.Google.Api.get_analytics_view_ids(site) + view_ids = Plausible.Google.Api.get_analytics_view_ids(access_token) conn |> assign(:skip_plausible_tracking, true) |> render("import_from_google.html", + access_token: access_token, site: site, view_ids: view_ids, layout: {PlausibleWeb.LayoutView, "focus.html"} ) end - def import_from_google(conn, %{"view_id" => view_id, "end_date" => end_date}) do - site = - conn.assigns[:site] - |> Repo.preload(:google_auth) + def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do + site = conn.assigns[:site] - cond do - site.imported_data -> - conn - |> put_flash(:error, "Data already imported from: #{site.imported_data.source}") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + redirect(conn, + to: + Routes.site_path(conn, :import_from_google_confirm, site.domain, + view_id: view_id, + access_token: access_token + ) + ) + end - true -> - job = - Plausible.Workers.ImportGoogleAnalytics.new(%{ - "site_id" => site.id, - "view_id" => view_id, - "end_date" => end_date - }) + def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do + site = conn.assigns[:site] - Ecto.Multi.new() - |> Ecto.Multi.update(:update_site, Plausible.Site.start_import(site, "Google Analytics")) - |> Oban.insert(:oban_job, job) - |> Repo.transaction() + # TODO: Plausible.Google.Api.get_analytics_start_date(access_token) + start_date = {:ok, ~D[2019-01-02]} + end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) - conn - |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) - end + conn + |> assign(:skip_plausible_tracking, true) + |> render("import_from_google_confirm.html", + access_token: access_token, + site: site, + # TODO: Also send the name here + view_id: view_id, + start_date: start_date, + end_date: end_date, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import_from_google(conn, %{ + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token + }) do + site = conn.assigns[:site] + + job = + Plausible.Workers.ImportGoogleAnalytics.new(%{ + "site_id" => site.id, + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token + }) + + Ecto.Multi.new() + |> Ecto.Multi.update( + :update_site, + Plausible.Site.start_import(site, start_date, end_date, "Google Analytics") + ) + |> Oban.insert(:oban_job, job) + |> Repo.transaction() + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) end def forget_imported(conn, _params) do diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index e925904e6..0da123cb2 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -22,6 +22,7 @@ defmodule PlausibleWeb.StatsController do |> render("stats.html", site: site, has_goals: Plausible.Sites.has_goals?(site), + stats_begin: Plausible.Sites.stats_begin(site), title: "Plausible · " <> site.domain, offer_email_report: offer_email_report, demo: demo @@ -177,6 +178,7 @@ defmodule PlausibleWeb.StatsController do |> render("stats.html", site: shared_link.site, has_goals: Plausible.Sites.has_goals?(shared_link.site), + stats_begin: Plausible.Sites.stats_begin(shared_link.site), title: "Plausible · " <> shared_link.site.domain, offer_email_report: false, demo: false, diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 1ef1e9b25..39f2b77b9 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -239,6 +239,8 @@ defmodule PlausibleWeb.Router do delete "/:website/stats", SiteController, :reset_stats get "/:website/import/google-analytics", SiteController, :import_from_google_form + post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id + get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm post "/:website/settings/google-import", SiteController, :import_from_google delete "/:website/settings/forget-imported", SiteController, :forget_imported diff --git a/lib/plausible_web/templates/site/import_from_google.html.eex b/lib/plausible_web/templates/site/import_from_google.html.eex index 441670302..04297e955 100644 --- a/lib/plausible_web/templates/site/import_from_google.html.eex +++ b/lib/plausible_web/templates/site/import_from_google.html.eex @@ -1,27 +1,22 @@ -<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>

Import from Google Analytics

+ <%= hidden_input(f, :access_token, value: @access_token) %> + <%= case @view_ids do %> <% {:ok, view_ids} -> %>
- You can choose the view ID that will be imported to the <%= @site.domain %> dashboard. + Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.
-
- <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %> - <%= select f, :view_id, view_ids, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> -
- -
- <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %> -
- <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %> -
+
+ <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700" %> + <%= select f, :view_id, view_ids, prompt: "(Choose view)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
<% {:error, error} -> %>

The following error occurred when fetching your Google Analytics view ids.

<%= error %>

<% end %> - <%= submit "Import", class: "button" %> + <%= submit "Continue ->", class: "button mt-6" %> <% end %> diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex new file mode 100644 index 000000000..2298054b5 --- /dev/null +++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex @@ -0,0 +1,38 @@ +<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +

Import from Google Analytics

+ + <%= hidden_input(f, :access_token, value: @access_token) %> + + <%= case @start_date do %> + <% {:ok, start_date} -> %> + +
+ Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard. +
+ +
+ <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700" %> + <%= text_input f, :view_id, value: @view_id, readonly: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> +
+
+ We will import historical data from your first Google Analytics visitor to your first Plausible visitor. If you'd + like to import data for a different time period, please contact support. +
+
+
+ <%= label f, :start_date, "From", class: "block text-sm font-medium text-gray-700" %> + <%= date_input f, :start_date, value: start_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> +
+
→
+
+ <%= label f, :end_date, "To", class: "block text-sm font-medium text-gray-700" %> + <%= date_input f, :end_date, value: @end_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> +
+
+ <% {:error, error} -> %> +

The following error occurred when fetching your Google Analytics data.

+

<%= error %>

+ <% end %> + + <%= submit "Confirm import", class: "button mt-6" %> +<% end %> diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index 51b24d3c3..83fee3166 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -58,73 +58,42 @@ <%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %> <%= cond do %> - <% @imported_data && @imported_data.status == "importing" -> %> + <% @site.imported_data && @site.imported_data.status == "importing" -> %>
-
We are importing data from <%= @imported_data.source %> in the background... You will receive an email when it's completed
+
We are importing data from <%= @site.imported_data.source %> in the background... You will receive an email when it's completed
- <% @imported_data && @imported_data.status == "ok" -> %> + <% @site.imported_data && @site.imported_data.status == "ok" -> %>

  • Forget Imported Data

    - Removes all data imported from <%= @imported_data.source %> + Removes all data imported from <%= @site.imported_data.source %>

    <%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %>
    • - <%= if @site.google_auth do %> - <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> - <% end %> - - <% @site.google_auth -> %> + <% @site.imported_data && @site.imported_data.status == "error" -> %>
    - Linked Google account: <%= @site.google_auth.email %> +
    Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.
    - <%= if @imported_data && @imported_data.status == "error" do %> -
    - Your last GA import resulted in an error. You can try again below. -
    - <% end %> - - <%= case @google_profiles do %> - <% {:ok, profiles} -> %> -

    - Select the Google Analytics view id you would like to import data from. -

    - - <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %> -
    - <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %> - <%= select f, :view_id, profiles, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> -
    - -
    - <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %> -
    - <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %> -
    -
    - - <%= submit "Import", class: "button" %> - <% end %> - - <% {:error, error} -> %> -

    The following error occurred when fetching your Google Analytics profiles.

    -

    <%= error %>

    - <% end %> - - <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> - - <% true -> %>
    - <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %> + <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %> <%= google_logo() %> Continue with Google <% end %>
    + + <% true -> %> +
    + <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %> + <%= google_logo() %> + Continue with Google + <% end %> +
    + <% end %> <% else %>
    diff --git a/lib/plausible_web/templates/stats/stats.html.eex b/lib/plausible_web/templates/stats/stats.html.eex index a47aac150..f6838654b 100644 --- a/lib/plausible_web/templates/stats/stats.html.eex +++ b/lib/plausible_web/templates/stats/stats.html.eex @@ -12,7 +12,7 @@ <% end %>
    -
    +
    <%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
    diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index b313891e1..6c26990ff 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -8,14 +8,20 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do @impl Oban.Worker def perform( - %Oban.Job{args: %{"site_id" => site_id, "view_id" => view_id, "end_date" => end_date}}, + %Oban.Job{ + args: %{ + "site_id" => site_id, + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token + } + }, google_api \\ Plausible.Google.Api ) do - site = - Repo.get(Plausible.Site, site_id) - |> Repo.preload([:google_auth, [memberships: :user]]) + site = Repo.get(Plausible.Site, site_id) |> Repo.preload([[memberships: :user]]) - case google_api.import_analytics(site, view_id, end_date) do + case google_api.import_analytics(site, view_id, start_date, end_date, access_token) do {:ok, _} -> Plausible.Site.import_success(site) |> Repo.update!() diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index dc3b475e5..838943558 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -727,26 +727,36 @@ defmodule PlausibleWeb.SiteControllerTest do test "adds in-progress imported tag to site", %{conn: conn, site: site} do post(conn, "/#{site.domain}/settings/google-import", %{ "view_id" => "123", - "end_date" => "2022-03-01" + "start_date" => "2018-03-01", + "end_date" => "2022-03-01", + "access_token" => "token" }) imported_data = Repo.reload(site).imported_data assert imported_data assert imported_data.source == "Google Analytics" - assert imported_data.end_date == Timex.today() + assert imported_data.end_date == ~D[2022-03-01] assert imported_data.status == "importing" end test "schedules an import job in Oban", %{conn: conn, site: site} do post(conn, "/#{site.domain}/settings/google-import", %{ "view_id" => "123", - "end_date" => "2022-03-01" + "start_date" => "2018-03-01", + "end_date" => "2022-03-01", + "access_token" => "token" }) assert_enqueued( worker: Plausible.Workers.ImportGoogleAnalytics, - args: %{"site_id" => site.id, "view_id" => "123", "end_date" => "2022-03-01"} + args: %{ + "site_id" => site.id, + "view_id" => "123", + "start_date" => "2018-03-01", + "end_date" => "2022-03-01", + "access_token" => "token" + } ) end end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index e0fdeb094..bb9076597 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -14,7 +14,7 @@ defmodule Plausible.TestUtils do def add_imported_data(%{site: site}) do site = site - |> Plausible.Site.start_import("Google Analytics", "ok") + |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok") |> Repo.update!() {:ok, site: site} diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exs index 63a06c47f..a26a5c58c 100644 --- a/test/workers/import_google_analytics_test.exs +++ b/test/workers/import_google_analytics_test.exs @@ -15,13 +15,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do site = insert(:site, members: [user], imported_data: @imported_data) api_stub = - stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date -> + stub(Plausible.Google.Api, :import_analytics, fn _site, + _view_id, + _start_date, + _end_date, + _access_token -> {:ok, nil} end) ImportGoogleAnalytics.perform( %Oban.Job{ - args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"} + args: %{ + "site_id" => site.id, + "view_id" => "view_id", + "start_date" => "2020-01-01", + "end_date" => "2022-01-01", + "access_token" => "token" + } }, api_stub ) @@ -34,13 +44,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do site = insert(:site, members: [user], imported_data: @imported_data) api_stub = - stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date -> + stub(Plausible.Google.Api, :import_analytics, fn _site, + _view_id, + _start_date, + _end_date, + _access_token -> {:ok, nil} end) ImportGoogleAnalytics.perform( %Oban.Job{ - args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"} + args: %{ + "site_id" => site.id, + "view_id" => "view_id", + "start_date" => "2020-01-01", + "end_date" => "2022-01-01", + "access_token" => "token" + } }, api_stub ) @@ -56,13 +76,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do site = insert(:site, members: [user], imported_data: @imported_data) api_stub = - stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date -> + stub(Plausible.Google.Api, :import_analytics, fn _site, + _view_id, + _start_date, + _end_date, + _access_token -> {:error, "Something went wrong"} end) ImportGoogleAnalytics.perform( %Oban.Job{ - args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"} + args: %{ + "site_id" => site.id, + "view_id" => "view_id", + "start_date" => "2020-01-01", + "end_date" => "2022-01-01", + "access_token" => "token" + } }, api_stub ) @@ -75,13 +105,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do site = insert(:site, members: [user], imported_data: @imported_data) api_stub = - stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date -> + stub(Plausible.Google.Api, :import_analytics, fn _site, + _view_id, + _start_date, + _end_date, + _access_token -> {:error, "Something went wrong"} end) ImportGoogleAnalytics.perform( %Oban.Job{ - args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"} + args: %{ + "site_id" => site.id, + "view_id" => "view_id", + "start_date" => "2020-01-01", + "end_date" => "2022-01-01", + "access_token" => "token" + } }, api_stub )