Ga import improvements (#1784)

* Do not link Google account for import

* Record start date

* Fix tests

assets/js/dashboard/datepicker.js

@@ -22,7 +22,7 @@ import {
 import { navigateToQuery, QueryLink, QueryButton } from "./query";
 
 function renderArrow(query, site, period, prevDate, nextDate) {
-  const insertionDate = parseUTCDate(site.insertedAt);
+  const insertionDate = parseUTCDate(site.statsBegin);
   const disabledLeft = isBefore(
     parseUTCDate(prevDate),
     insertionDate,
@@ -136,7 +136,7 @@ class DatePicker extends React.Component {
       date: false,
     };
 
-    const insertionDate = parseUTCDate(this.props.site.insertedAt);
+    const insertionDate = parseUTCDate(this.props.site.statsBegin);
 
     if (e.key === "ArrowLeft") {
       const prevDate = formatISO(shiftDays(query.date, -1));
@@ -358,7 +358,7 @@ class DatePicker extends React.Component {
         </div>
       );
     } if (this.state.mode === "calendar") {
-      const insertionDate = new Date(this.props.site.insertedAt);
+      const insertionDate = new Date(this.props.site.statsBegin);
       const dayBeforeCreation = insertionDate - 86400000;
       return (
         <div className="h-0">

assets/js/dashboard/mount.js

@@ -13,7 +13,7 @@ if (container) {
     domain: container.dataset.domain,
     offset: container.dataset.offset,
     hasGoals: container.dataset.hasGoals === 'true',
-    insertedAt: container.dataset.insertedAt,
+    statsBegin: container.dataset.statsBegin,
     embedded: container.dataset.embedded,
     background: container.dataset.background,
     selfhosted: container.dataset.selfhosted === 'true'

lib/mix/tasks/call_google.ex

@@ -1,48 +0,0 @@
-defmodule Mix.Tasks.CallGoogle do
-  use Mix.Task
-  use Plausible.Repo
-  require Logger
-
-  @token "ya29.A0ARrdaM-GC-C38oUNmoUlica3v2IGT1eACtvreAFcM4zCYrpQxHdTFfhpmWQfiYAblnKPVRkt_fW-jTTwJ_ScXtkaqq6P_XiBbPIa3g9yJ_F6bP2tEOeHgq07aPlPwHvqXaK8I8gmwtO86VvJLvmhXolHtlXN0g"
-  @view_id "204615196"
-  @dimensions ["ga:date"]
-  @metrics ["ga:users", "ga:pageviews", "ga:bounces", "ga:sessions", "ga:sessionDuration"]
-
-  def run(_) do
-    Mix.Task.run("app.start")
-
-    report = %{
-      viewId: @view_id,
-      dateRanges: [
-        %{
-          # The earliest valid date
-          startDate: "2005-01-01",
-          endDate: "2019-12-05"
-        }
-      ],
-      dimensions: Enum.map(@dimensions, &%{name: &1, histogramBuckets: []}),
-      metrics: Enum.map(@metrics, &%{expression: &1}),
-      hideTotals: true,
-      hideValueRanges: true,
-      orderBys: [
-        %{
-          fieldName: "ga:date",
-          sortOrder: "DESCENDING"
-        }
-      ],
-      pageSize: 100_00,
-      pageToken: ""
-    }
-
-    Logger.debug(report)
-
-    res =
-      HTTPoison.post!(
-        "https://analyticsreporting.googleapis.com/v4/reports:batchGet",
-        Jason.encode!(%{reportRequests: [report]}),
-        Authorization: "Bearer #{@token}"
-      )
-
-    Logger.debug(res.body)
-  end
-end

lib/plausible/google/api.ex

@@ -6,6 +6,7 @@ defmodule Plausible.Google.Api do
   @scope URI.encode_www_form(
            "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly"
          )
+  @import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly")
   @verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"]
 
   def authorize_url(site_id, redirect_to) do
@@ -17,6 +18,15 @@ defmodule Plausible.Google.Api do
     end
   end
 
+  def import_authorize_url(site_id, redirect_to) do
+    if Application.get_env(:plausible, :environment) == "test" do
+      ""
+    else
+      "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
+        Jason.encode!([site_id, redirect_to])
+    end
+  end
+
   def fetch_access_token(code) do
     res =
       HTTPoison.post!(
@@ -120,17 +130,11 @@ defmodule Plausible.Google.Api do
     end
   end
 
-  def get_analytics_view_ids(site) do
-    with {:ok, auth} <- refresh_if_needed(site.google_auth) do
-      do_get_analytics_view_ids(auth)
-    end
-  end
-
-  def do_get_analytics_view_ids(auth) do
+  def get_analytics_view_ids(token) do
     res =
       HTTPoison.get!(
         "https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles",
-        Authorization: "Bearer #{auth.access_token}"
+        Authorization: "Bearer #{token}"
       )
 
     case res.status_code do
@@ -153,11 +157,60 @@ defmodule Plausible.Google.Api do
     end
   end
 
-  def import_analytics(site, view_id, end_date) do
-    with {:ok, auth} <- refresh_if_needed(site.google_auth) do
-      do_import_analytics(site, auth, view_id, end_date)
-    end
-  end
+  # Each element is: {dataset, dimensions, metrics}
+  @request_data [
+    {
+      "imported_visitors",
+      ["ga:date"],
+      [
+        "ga:users",
+        "ga:pageviews",
+        "ga:bounces",
+        "ga:sessions",
+        "ga:sessionDuration"
+      ]
+    },
+    {
+      "imported_sources",
+      ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"],
+      ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
+    },
+    {
+      "imported_pages",
+      ["ga:date", "ga:hostname", "ga:pagePath"],
+      ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]
+    },
+    {
+      "imported_entry_pages",
+      ["ga:date", "ga:landingPagePath"],
+      ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]
+    },
+    {
+      "imported_exit_pages",
+      ["ga:date", "ga:exitPagePath"],
+      ["ga:users", "ga:exits"]
+    },
+    {
+      "imported_locations",
+      ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],
+      ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
+    },
+    {
+      "imported_devices",
+      ["ga:date", "ga:deviceCategory"],
+      ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
+    },
+    {
+      "imported_browsers",
+      ["ga:date", "ga:browser"],
+      ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
+    },
+    {
+      "imported_operating_systems",
+      ["ga:date", "ga:operatingSystem"],
+      ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
+    }
+  ]
 
   @doc """
   API reference:
@@ -165,71 +218,17 @@ defmodule Plausible.Google.Api do
 
   Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer
   """
-  def do_import_analytics(site, auth, view_id, end_date) do
+  def import_analytics(site, view_id, start_date, end_date, access_token) do
     request = %{
-      auth: auth,
-      profile: view_id,
+      access_token: access_token,
+      view_id: view_id,
+      start_date: start_date,
       end_date: end_date
     }
 
-    # Each element is: {dataset, dimensions, metrics}
-    request_data = [
-      {
-        "imported_visitors",
-        ["ga:date"],
-        [
-          "ga:users",
-          "ga:pageviews",
-          "ga:bounces",
-          "ga:sessions",
-          "ga:sessionDuration"
-        ]
-      },
-      {
-        "imported_sources",
-        ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"],
-        ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
-      },
-      {
-        "imported_pages",
-        ["ga:date", "ga:hostname", "ga:pagePath"],
-        ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]
-      },
-      {
-        "imported_entry_pages",
-        ["ga:date", "ga:landingPagePath"],
-        ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]
-      },
-      {
-        "imported_exit_pages",
-        ["ga:date", "ga:exitPagePath"],
-        ["ga:users", "ga:exits"]
-      },
-      {
-        "imported_locations",
-        ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],
-        ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
-      },
-      {
-        "imported_devices",
-        ["ga:date", "ga:deviceCategory"],
-        ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
-      },
-      {
-        "imported_browsers",
-        ["ga:date", "ga:browser"],
-        ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
-      },
-      {
-        "imported_operating_systems",
-        ["ga:date", "ga:operatingSystem"],
-        ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
-      }
-    ]
-
     responses =
       Enum.map(
-        request_data,
+        @request_data,
         fn {dataset, dimensions, metrics} ->
           fetch_analytic_reports(dataset, dimensions, metrics, request)
         end
@@ -272,11 +271,11 @@ defmodule Plausible.Google.Api do
 
   defp fetch_analytic_reports(dataset, dimensions, metrics, request, page_token \\ "") do
     report = %{
-      viewId: request.profile,
+      viewId: request.view_id,
       dateRanges: [
         %{
           # The earliest valid date
-          startDate: "2005-01-01",
+          startDate: request.start_date,
           endDate: request.end_date
         }
       ],
@@ -298,7 +297,7 @@ defmodule Plausible.Google.Api do
       HTTPoison.post!(
         "https://analyticsreporting.googleapis.com/v4/reports:batchGet",
         Jason.encode!(%{reportRequests: [report]}),
-        [Authorization: "Bearer #{request.auth.access_token}"],
+        [Authorization: "Bearer #{request.access_token}"],
         timeout: 30_000,
         recv_timeout: 30_000
       )

lib/plausible/site/schema.ex

@@ -2,6 +2,7 @@ defmodule Plausible.Site.ImportedData do
   use Ecto.Schema
 
   embedded_schema do
+    field :start_date, :date
     field :end_date, :date
     field :source, :string
     field :status, :string
@@ -62,10 +63,11 @@ defmodule Plausible.Site do
     change(site, has_stats: has_stats_val)
   end
 
-  def start_import(site, imported_source, status \\ "importing") do
+  def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
     change(site,
-      imported_data: %Plausible.Site.ImportedData{
-        end_date: Timex.today(),
+      imported_data: %{
+        start_date: start_date,
+        end_date: end_date,
         source: imported_source,
         status: status
       }

lib/plausible/sites.ex

@@ -55,6 +55,14 @@ defmodule Plausible.Sites do
     end
   end
 
+  def stats_begin(site) do
+    if site.imported_data do
+      site.imported_data.start_date
+    else
+      site.inserted_at
+    end
+  end
+
   def create_shared_link(site, name, password \\ nil) do
     changes =
       SharedLink.changeset(

lib/plausible/stats/clickhouse.ex

@@ -5,6 +5,24 @@ defmodule Plausible.Stats.Clickhouse do
   use Plausible.Stats.Fragments
   @no_ref "Direct / None"
 
+  def pageview_start_date_local(site) do
+    date =
+      ClickhouseRepo.one(
+        from e in "events",
+          select: fragment("toDate(min(?))", e.timestamp),
+          where: e.domain == ^site.domain
+      )
+
+    case date do
+      # no stats for this domain yet
+      ~D[1970-01-01] ->
+        Timex.today(site.timezone)
+
+      date ->
+        Timex.Timezone.convert(date, site.timezone)
+    end
+  end
+
   def usage_breakdown(domains) do
     range =
       Date.range(

lib/plausible_web/controllers/auth_controller.ex

@@ -538,24 +538,36 @@ defmodule PlausibleWeb.AuthController do
 
   def google_auth_callback(conn, %{"code" => code, "state" => state}) do
     res = Plausible.Google.Api.fetch_access_token(code)
-    id_token = res["id_token"]
-    [_, body, _] = String.split(id_token, ".")
-    id = body |> Base.decode64!(padding: false) |> Jason.decode!()
-
     [site_id, redirect_to] = Jason.decode!(state)
-
-    Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{
-      email: id["email"],
-      refresh_token: res["refresh_token"],
-      access_token: res["access_token"],
-      expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),
-      user_id: conn.assigns[:current_user].id,
-      site_id: site_id
-    })
-    |> Repo.insert!()
-
     site = Repo.get(Plausible.Site, site_id)
 
-    redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}")
+    case redirect_to do
+      "import" ->
+        redirect(conn,
+          to:
+            Routes.site_path(conn, :import_from_google_form, site.domain,
+              access_token: res["access_token"]
+            )
+        )
+
+      _ ->
+        id_token = res["id_token"]
+        [_, body, _] = String.split(id_token, ".")
+        id = body |> Base.decode64!(padding: false) |> Jason.decode!()
+
+        Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{
+          email: id["email"],
+          refresh_token: res["refresh_token"],
+          access_token: res["access_token"],
+          expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),
+          user_id: conn.assigns[:current_user].id,
+          site_id: site_id
+        })
+        |> Repo.insert!()
+
+        site = Repo.get(Plausible.Site, site_id)
+
+        redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}")
+    end
   end
 end

lib/plausible_web/controllers/site_controller.ex

@@ -167,28 +167,15 @@ defmodule PlausibleWeb.SiteController do
     redirect(conn, to: Routes.site_path(conn, :settings_general, website))
   end
 
-  defp can_trigger_import(site) do
-    no_import = is_nil(site.imported_data) || site.imported_data.status == "error"
-
-    no_import && site.google_auth
-  end
-
   def settings_general(conn, _params) do
     site =
       conn.assigns[:site]
       |> Repo.preload([:custom_domain, :google_auth])
 
-    google_profiles =
-      if can_trigger_import(site) do
-        Plausible.Google.Api.get_analytics_view_ids(site)
-      end
-
     conn
     |> assign(:skip_plausible_tracking, true)
     |> render("settings_general.html",
       site: site,
-      google_profiles: google_profiles,
-      imported_data: site.imported_data,
       changeset: Plausible.Site.changeset(site, %{}),
       layout: {PlausibleWeb.LayoutView, "site_settings.html"}
     )
@@ -643,48 +630,81 @@ defmodule PlausibleWeb.SiteController do
     |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings/general")
   end
 
-  def import_from_google_form(conn, _params) do
-    site = conn.assigns[:site] |> Repo.preload(:google_auth)
+  def import_from_google_form(conn, %{"access_token" => access_token}) do
+    site = conn.assigns[:site]
 
-    view_ids = Plausible.Google.Api.get_analytics_view_ids(site)
+    view_ids = Plausible.Google.Api.get_analytics_view_ids(access_token)
 
     conn
     |> assign(:skip_plausible_tracking, true)
     |> render("import_from_google.html",
+      access_token: access_token,
       site: site,
       view_ids: view_ids,
       layout: {PlausibleWeb.LayoutView, "focus.html"}
     )
   end
 
-  def import_from_google(conn, %{"view_id" => view_id, "end_date" => end_date}) do
-    site =
-      conn.assigns[:site]
-      |> Repo.preload(:google_auth)
+  def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do
+    site = conn.assigns[:site]
 
-    cond do
-      site.imported_data ->
-        conn
-        |> put_flash(:error, "Data already imported from: #{site.imported_data.source}")
-        |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))
+    redirect(conn,
+      to:
+        Routes.site_path(conn, :import_from_google_confirm, site.domain,
+          view_id: view_id,
+          access_token: access_token
+        )
+    )
+  end
 
-      true ->
-        job =
-          Plausible.Workers.ImportGoogleAnalytics.new(%{
-            "site_id" => site.id,
-            "view_id" => view_id,
-            "end_date" => end_date
-          })
+  def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do
+    site = conn.assigns[:site]
 
-        Ecto.Multi.new()
-        |> Ecto.Multi.update(:update_site, Plausible.Site.start_import(site, "Google Analytics"))
-        |> Oban.insert(:oban_job, job)
-        |> Repo.transaction()
+    # TODO: Plausible.Google.Api.get_analytics_start_date(access_token)
+    start_date = {:ok, ~D[2019-01-02]}
+    end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)
 
-        conn
-        |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
-        |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))
-    end
+    conn
+    |> assign(:skip_plausible_tracking, true)
+    |> render("import_from_google_confirm.html",
+      access_token: access_token,
+      site: site,
+      # TODO: Also send the name here
+      view_id: view_id,
+      start_date: start_date,
+      end_date: end_date,
+      layout: {PlausibleWeb.LayoutView, "focus.html"}
+    )
+  end
+
+  def import_from_google(conn, %{
+        "view_id" => view_id,
+        "start_date" => start_date,
+        "end_date" => end_date,
+        "access_token" => access_token
+      }) do
+    site = conn.assigns[:site]
+
+    job =
+      Plausible.Workers.ImportGoogleAnalytics.new(%{
+        "site_id" => site.id,
+        "view_id" => view_id,
+        "start_date" => start_date,
+        "end_date" => end_date,
+        "access_token" => access_token
+      })
+
+    Ecto.Multi.new()
+    |> Ecto.Multi.update(
+      :update_site,
+      Plausible.Site.start_import(site, start_date, end_date, "Google Analytics")
+    )
+    |> Oban.insert(:oban_job, job)
+    |> Repo.transaction()
+
+    conn
+    |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
+    |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))
   end
 
   def forget_imported(conn, _params) do

lib/plausible_web/controllers/stats_controller.ex

@@ -22,6 +22,7 @@ defmodule PlausibleWeb.StatsController do
         |> render("stats.html",
           site: site,
           has_goals: Plausible.Sites.has_goals?(site),
+          stats_begin: Plausible.Sites.stats_begin(site),
           title: "Plausible · " <> site.domain,
           offer_email_report: offer_email_report,
           demo: demo
@@ -177,6 +178,7 @@ defmodule PlausibleWeb.StatsController do
         |> render("stats.html",
           site: shared_link.site,
           has_goals: Plausible.Sites.has_goals?(shared_link.site),
+          stats_begin: Plausible.Sites.stats_begin(shared_link.site),
           title: "Plausible · " <> shared_link.site.domain,
           offer_email_report: false,
           demo: false,

lib/plausible_web/router.ex

@@ -239,6 +239,8 @@ defmodule PlausibleWeb.Router do
     delete "/:website/stats", SiteController, :reset_stats
 
     get "/:website/import/google-analytics", SiteController, :import_from_google_form
+    post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id
+    get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm
     post "/:website/settings/google-import", SiteController, :import_from_google
     delete "/:website/settings/forget-imported", SiteController, :forget_imported
 

lib/plausible_web/templates/site/import_from_google.html.eex

@@ -1,27 +1,22 @@
-<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
   <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
 
+  <%= hidden_input(f, :access_token, value: @access_token) %>
+
   <%= case @view_ids do %>
     <% {:ok, view_ids} -> %>
 
       <div class="mt-6 text-sm text-gray-500">
-        You can choose the view ID that will be imported to the <%= @site.domain %> dashboard.
+        Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.
       </div>
 
-      <div class="my-6">
-        <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700"  %>
-        <%= select f, :view_id, view_ids, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
-      </div>
-
-      <div class="my-6">
-        <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %>
-        <div class="mt-1">
-          <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>
-        </div>
+      <div class="mt-3">
+        <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700"  %>
+        <%= select f, :view_id, view_ids, prompt: "(Choose view)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
       </div>
     <% {:error, error} -> %>
       <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics view ids.</p>
       <p class="text-red-700 font-medium mt-3"><%= error %></p>
     <% end %>
-  <%= submit "Import", class: "button" %>
+    <%= submit "Continue ->", class: "button mt-6" %>
 <% end %>

lib/plausible_web/templates/site/import_from_google_confirm.html.eex

@@ -0,0 +1,38 @@
+<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+  <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
+
+  <%= hidden_input(f, :access_token, value: @access_token) %>
+
+  <%= case @start_date do %>
+    <% {:ok, start_date} -> %>
+
+      <div class="mt-6 text-sm text-gray-500">
+        Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.
+      </div>
+
+      <div class="mt-3">
+        <%= label f, :view_id, "Google Analytics view", class: "block text-sm font-medium text-gray-700"  %>
+        <%= text_input f, :view_id, value: @view_id, readonly: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
+      </div>
+      <div class="mt-6 text-sm text-gray-500">
+        We will import historical data from your first Google Analytics visitor to your first Plausible visitor. If you'd
+        like to import data for a different time period, please contact support.
+      </div>
+      <div class="flex justify-between mt-3">
+        <div class="w-36">
+          <%= label f, :start_date, "From", class: "block text-sm font-medium text-gray-700"  %>
+          <%= date_input f, :start_date, value: start_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
+        </div>
+        <div class="align-middle pt-8">&rarr;</div>
+        <div class="w-36">
+          <%= label f, :end_date, "To", class: "block text-sm font-medium text-gray-700"  %>
+          <%= date_input f, :end_date, value: @end_date, readonly: "true", class: "mt-1 block w-full px-3 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
+        </div>
+      </div>
+    <% {:error, error} -> %>
+      <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics data.</p>
+      <p class="text-red-700 font-medium mt-3"><%= error %></p>
+    <% end %>
+
+  <%= submit "Confirm import", class: "button mt-6" %>
+<% end %>

lib/plausible_web/templates/site/settings_general.html.eex

@@ -58,73 +58,42 @@
 
   <%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %>
     <%= cond do %>
-      <% @imported_data && @imported_data.status == "importing" -> %>
+      <% @site.imported_data && @site.imported_data.status == "importing" -> %>
         <div class="py-2"></div>
-        <div class="text-sm">We are importing data from <%= @imported_data.source %> in the background... You will receive an email when it's completed</div>
+        <div class="text-sm">We are importing data from <%= @site.imported_data.source %> in the background... You will receive an email when it's completed</div>
 
-      <% @imported_data && @imported_data.status == "ok" -> %>
+      <% @site.imported_data && @site.imported_data.status == "ok" -> %>
         <li class="py-4 flex items-center justify-between space-x-4">
           <div class="flex flex-col">
             <p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
             Forget Imported Data
             </p>
             <p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
-            Removes all data imported from <%= @imported_data.source %>
+            Removes all data imported from <%= @site.imported_data.source %>
             </p>
           </div>
           <%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %>
         </li>
 
-        <%= if @site.google_auth do %>
-          <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %>
-        <% end %>
-
-      <% @site.google_auth -> %>
+      <% @site.imported_data && @site.imported_data.status == "error" -> %>
         <div class="py-2"></div>
-        <span class="text-gray-700 dark:text-gray-300">Linked Google account: <b><%= @site.google_auth.email %></b></span>
+        <div class="text-sm">Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.</div>
 
-        <%= if @imported_data && @imported_data.status == "error" do %>
-          <div class="mt-1 text-red-600">
-            Your last GA import resulted in an error. You can try again below.
-          </div>
-        <% end %>
-
-        <%= case @google_profiles do %>
-          <% {:ok, profiles} -> %>
-            <p class="text-gray-700 dark:text-gray-300 mt-1">
-            Select the Google Analytics view id you would like to import data from.
-            </p>
-
-            <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %>
-              <div class="my-6">
-                <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700"  %>
-                <%= select f, :view_id, profiles, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %>
-              </div>
-
-              <div class="my-6">
-                <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %>
-                <div class="mt-1">
-                  <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>
-                </div>
-              </div>
-
-              <%= submit "Import", class: "button" %>
-            <% end %>
-
-          <% {:error, error} -> %>
-            <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics profiles.</p>
-            <p class="text-red-700 font-medium mt-3"><%= error %></p>
-        <% end %>
-
-        <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %>
-
-      <% true -> %>
         <div class="flex">
-          <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %>
+          <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %>
             <%= google_logo() %>
             <span style="font-family: Roboto, system-ui" class="text-sm font-medium text-gray-600">Continue with Google<span>
           <% end %>
         </div>
+
+      <% true -> %>
+        <div class="flex">
+          <%= button(to: Plausible.Google.Api.authorize_url(@site.id, "import"), class: "inline-flex pr-4 items-center border border-gray-100 shadow rounded-md focus:outline-none focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50") do %>
+            <%= google_logo() %>
+            <span style="font-family: Roboto, system-ui" class="text-sm font-medium text-gray-600">Continue with Google<span>
+          <% end %>
+        </div>
+
     <% end %>
   <% else %>
     <div class="my-8 text-center text-lg">

lib/plausible_web/templates/stats/stats.html.eex

@@ -12,7 +12,7 @@
   <% end %>
 
   <div class="pt-6"></div>
-  <div id="stats-react-container" style="overflow-anchor: none;" data-domain="<%= @site.domain %>" data-offset="<%= Timex.Timezone.total_offset(Timex.Timezone.get(@site.timezone)) %>" data-has-goals="<%= @has_goals %>" data-logged-in="<%= !!@conn.assigns[:current_user] %>" data-inserted-at="<%= @site.inserted_at %>" data-shared-link-auth="<%= assigns[:shared_link_auth] %>" data-embedded="<%= @conn.assigns[:embedded] %>" data-background="<%= @conn.assigns[:background] %>" data-selfhosted="<%= Application.get_env(:plausible, :is_selfhost) %>" data-current-user-role="<%= @conn.assigns[:current_user_role] %>"></div>
+  <div id="stats-react-container" style="overflow-anchor: none;" data-domain="<%= @site.domain %>" data-offset="<%= Timex.Timezone.total_offset(Timex.Timezone.get(@site.timezone)) %>" data-has-goals="<%= @has_goals %>" data-logged-in="<%= !!@conn.assigns[:current_user] %>" data-stats-begin="<%= @stats_begin %>" data-shared-link-auth="<%= assigns[:shared_link_auth] %>" data-embedded="<%= @conn.assigns[:embedded] %>" data-background="<%= @conn.assigns[:background] %>" data-selfhosted="<%= Application.get_env(:plausible, :is_selfhost) %>" data-current-user-role="<%= @conn.assigns[:current_user_role] %>"></div>
   <div id="modal_root"></div>
   <%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
     <div class="bg-gray-50 dark:bg-gray-850">

lib/workers/import_google_analytics.ex

@@ -8,14 +8,20 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
 
   @impl Oban.Worker
   def perform(
-        %Oban.Job{args: %{"site_id" => site_id, "view_id" => view_id, "end_date" => end_date}},
+        %Oban.Job{
+          args: %{
+            "site_id" => site_id,
+            "view_id" => view_id,
+            "start_date" => start_date,
+            "end_date" => end_date,
+            "access_token" => access_token
+          }
+        },
         google_api \\ Plausible.Google.Api
       ) do
-    site =
-      Repo.get(Plausible.Site, site_id)
-      |> Repo.preload([:google_auth, [memberships: :user]])
+    site = Repo.get(Plausible.Site, site_id) |> Repo.preload([[memberships: :user]])
 
-    case google_api.import_analytics(site, view_id, end_date) do
+    case google_api.import_analytics(site, view_id, start_date, end_date, access_token) do
       {:ok, _} ->
         Plausible.Site.import_success(site)
         |> Repo.update!()

test/plausible_web/controllers/site_controller_test.exs

@@ -727,26 +727,36 @@ defmodule PlausibleWeb.SiteControllerTest do
     test "adds in-progress imported tag to site", %{conn: conn, site: site} do
       post(conn, "/#{site.domain}/settings/google-import", %{
         "view_id" => "123",
-        "end_date" => "2022-03-01"
+        "start_date" => "2018-03-01",
+        "end_date" => "2022-03-01",
+        "access_token" => "token"
       })
 
       imported_data = Repo.reload(site).imported_data
 
       assert imported_data
       assert imported_data.source == "Google Analytics"
-      assert imported_data.end_date == Timex.today()
+      assert imported_data.end_date == ~D[2022-03-01]
       assert imported_data.status == "importing"
     end
 
     test "schedules an import job in Oban", %{conn: conn, site: site} do
       post(conn, "/#{site.domain}/settings/google-import", %{
         "view_id" => "123",
-        "end_date" => "2022-03-01"
+        "start_date" => "2018-03-01",
+        "end_date" => "2022-03-01",
+        "access_token" => "token"
       })
 
       assert_enqueued(
         worker: Plausible.Workers.ImportGoogleAnalytics,
-        args: %{"site_id" => site.id, "view_id" => "123", "end_date" => "2022-03-01"}
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "123",
+          "start_date" => "2018-03-01",
+          "end_date" => "2022-03-01",
+          "access_token" => "token"
+        }
       )
     end
   end

test/support/test_utils.ex

@@ -14,7 +14,7 @@ defmodule Plausible.TestUtils do
   def add_imported_data(%{site: site}) do
     site =
       site
-      |> Plausible.Site.start_import("Google Analytics", "ok")
+      |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
       |> Repo.update!()
 
     {:ok, site: site}

test/workers/import_google_analytics_test.exs

@@ -15,13 +15,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
     site = insert(:site, members: [user], imported_data: @imported_data)
 
     api_stub =
-      stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->
+      stub(Plausible.Google.Api, :import_analytics, fn _site,
+                                                       _view_id,
+                                                       _start_date,
+                                                       _end_date,
+                                                       _access_token ->
         {:ok, nil}
       end)
 
     ImportGoogleAnalytics.perform(
       %Oban.Job{
-        args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "view_id",
+          "start_date" => "2020-01-01",
+          "end_date" => "2022-01-01",
+          "access_token" => "token"
+        }
       },
       api_stub
     )
@@ -34,13 +44,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
     site = insert(:site, members: [user], imported_data: @imported_data)
 
     api_stub =
-      stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->
+      stub(Plausible.Google.Api, :import_analytics, fn _site,
+                                                       _view_id,
+                                                       _start_date,
+                                                       _end_date,
+                                                       _access_token ->
         {:ok, nil}
       end)
 
     ImportGoogleAnalytics.perform(
       %Oban.Job{
-        args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "view_id",
+          "start_date" => "2020-01-01",
+          "end_date" => "2022-01-01",
+          "access_token" => "token"
+        }
       },
       api_stub
     )
@@ -56,13 +76,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
     site = insert(:site, members: [user], imported_data: @imported_data)
 
     api_stub =
-      stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->
+      stub(Plausible.Google.Api, :import_analytics, fn _site,
+                                                       _view_id,
+                                                       _start_date,
+                                                       _end_date,
+                                                       _access_token ->
         {:error, "Something went wrong"}
       end)
 
     ImportGoogleAnalytics.perform(
       %Oban.Job{
-        args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "view_id",
+          "start_date" => "2020-01-01",
+          "end_date" => "2022-01-01",
+          "access_token" => "token"
+        }
       },
       api_stub
     )
@@ -75,13 +105,23 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
     site = insert(:site, members: [user], imported_data: @imported_data)
 
     api_stub =
-      stub(Plausible.Google.Api, :import_analytics, fn _site, _view_id, _end_date ->
+      stub(Plausible.Google.Api, :import_analytics, fn _site,
+                                                       _view_id,
+                                                       _start_date,
+                                                       _end_date,
+                                                       _access_token ->
         {:error, "Something went wrong"}
       end)
 
     ImportGoogleAnalytics.perform(
       %Oban.Job{
-        args: %{"site_id" => site.id, "view_id" => "view_id", "end_date" => "2022-01-01"}
+        args: %{
+          "site_id" => site.id,
+          "view_id" => "view_id",
+          "start_date" => "2020-01-01",
+          "end_date" => "2022-01-01",
+          "access_token" => "token"
+        }
       },
       api_stub
     )