plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-12-22 09:01:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove references to site.imported_data (#4006)

Browse Source 
* Remove references to `site.imported_data`

* Count pre-existing ID 0 imports when showing pageview count summary for legacy imports

* Fix tests after rebase

* Dry `delete_imported_stats!`

* Clean up remaining imported data references and add notes
This commit is contained in:
Adrian Gruntkowski 2024-04-19 11:15:51 +02:00 committed by GitHub
parent 83643450c4
commit c10580777e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 272 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib/plausible/data_migration/site_imports.ex
View File

@ -58,7 +58,7 @@ defmodule Plausible.DataMigration.SiteImports do
params =
site.imported_data
|> Imported.SiteImport.from_legacy()
|> from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
@ -216,4 +216,22 @@ defmodule Plausible.DataMigration.SiteImports do
select: %{max_date: fragment("max(?)", q.date)}
)
end
defp from_legacy(%Site.ImportedData{} = data) do
status =
case data.status do
"ok" -> SiteImport.completed()
"error" -> SiteImport.failed()
_ -> SiteImport.importing()
end
%SiteImport{
id: 0,
legacy: true,
start_date: data.start_date,
end_date: data.end_date,
source: :universal_analytics,
status: status
}
end
end

43
lib/plausible/imported.ex
View File

@ -67,20 +67,23 @@ defmodule Plausible.Imported do
Repo.get_by(SiteImport, id: import_id, site_id: site.id)
end
@spec get_legacy_import(Site.t()) :: SiteImport.t() | nil
def get_legacy_import(site) do
from(i in SiteImport,
where: i.site_id == ^site.id and i.legacy == true,
order_by: [desc: i.updated_at],
limit: 1
)
|> Repo.one()
end
defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
def list_all_imports(site, status \\ nil) do
imports =
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|> maybe_filter_by_status(status)
|> Repo.all()
if site.imported_data && not Enum.any?(imports, & &1.legacy) do
imports ++ [SiteImport.from_legacy(site.imported_data)]
else
imports
end
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|> maybe_filter_by_status(status)
|> Repo.all()
end
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
@ -113,7 +116,7 @@ defmodule Plausible.Imported do
ids = Enum.map(ids, &elem(&1, 1))
# account for legacy imports as well
if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do
if has_legacy? do
[0 | ids]
else
ids
@ -132,23 +135,7 @@ defmodule Plausible.Imported do
)
|> Repo.one()
dates = dates || %{start_date: nil, end_date: nil}
if site.imported_data && site.imported_data.status == "ok" do
start_date =
[dates.start_date, site.imported_data.start_date]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
end_date =
[dates.end_date, site.imported_data.end_date]
|> Enum.reject(&is_nil/1)
|> Enum.max(Date, fn -> nil end)
%{start_date: start_date, end_date: end_date}
else
dates
end
dates || %{start_date: nil, end_date: nil}
end
@spec delete_imports_for_site(Site.t()) :: :ok

27
lib/plausible/imported/site_import.ex
View File

@ -33,34 +33,11 @@ defmodule Plausible.Imported.SiteImport do
defmacro unquote(status)(), do: unquote(status)
end
@spec label(t() | Site.ImportedData.t()) :: String.t()
def label(%__MODULE__{source: source, label: label}) do
@spec label(t()) :: String.t()
def label(%{source: source, label: label}) do
build_label(ImportSources.by_name(source).label(), label)
end
# NOTE: this is necessary for backwards compatibility
# with legacy imports
def label(%Site.ImportedData{source: source}), do: build_label(source, nil)
@spec from_legacy(Site.ImportedData.t()) :: t()
def from_legacy(%Site.ImportedData{} = data) do
status =
case data.status do
"ok" -> completed()
"error" -> failed()
_ -> importing()
end
%__MODULE__{
id: 0,
legacy: true,
start_date: data.start_date,
end_date: data.end_date,
source: :universal_analytics,
status: status
}
end
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
def create_changeset(site, user, params) do
%__MODULE__{}

45
lib/plausible/imported/universal_analytics.ex
View File

@ -5,8 +5,6 @@ defmodule Plausible.Imported.UniversalAnalytics do
use Plausible.Imported.Importer
alias Plausible.Repo
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@impl true
@ -18,49 +16,6 @@ defmodule Plausible.Imported.UniversalAnalytics do
@impl true
def email_template(), do: "google_analytics_import.html"
@impl true
def before_start(site_import) do
if site_import.legacy do
site = Repo.preload(site_import, :site).site
site
|> Plausible.Site.start_import(
site_import.start_date,
site_import.end_date,
label()
)
|> Repo.update!()
end
:ok
end
@impl true
def on_success(site_import, _extra_data) do
if site_import.legacy do
site = Repo.preload(site_import, :site).site
site
|> Plausible.Site.import_success()
|> Repo.update!()
end
:ok
end
@impl true
def on_failure(site_import) do
if site_import.legacy do
site = Repo.preload(site_import, :site).site
site
|> Plausible.Site.import_failure()
|> Repo.update!()
end
:ok
end
@impl true
def parse_args(
%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = args

4
lib/plausible/purge.ex
View File

@ -60,6 +60,10 @@ defmodule Plausible.Purge do
site_import = Repo.preload(site_import, :site)
delete_imported_stats!(site_import.site, site_import.id)
if site_import.legacy do
delete_imported_stats!(site_import.site, 0)
end
:ok
end

26
lib/plausible/site.ex
View File

@ -29,6 +29,7 @@ defmodule Plausible.Site do
field :domain_changed_from, :string
field :domain_changed_at, :naive_datetime
# NOTE: needed by `SiteImports` data migration script
embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
many_to_many :members, User, join_through: Plausible.Site.Membership
@ -153,31 +154,6 @@ defmodule Plausible.Site do
change(site, native_stats_start_at: val)
end
def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
change(site,
imported_data: %{
start_date: start_date,
end_date: end_date,
source: imported_source,
status: status
}
)
end
def import_success(site) do
change(site,
imported_data: %{status: "ok"}
)
end
def import_failure(site) do
change(site, imported_data: %{status: "error"})
end
def remove_imported_data(site) do
change(site, imported_data: nil)
end
defp clean_domain(changeset) do
clean_domain =
(get_field(changeset, :domain) || "")

2
lib/plausible/site/imported_data.ex
View File

@ -1,6 +1,8 @@
defmodule Plausible.Site.ImportedData do
@moduledoc """
Embedded schema for analytics imports
NOTE: needed by `SiteImports` data migration script
"""
use Ecto.Schema

4
lib/plausible_web/components/settings.ex
View File

@ -7,6 +7,10 @@ defmodule PlausibleWeb.Components.Settings do
import PlausibleWeb.Components.Generic
alias Plausible.Imported.SiteImport
require Plausible.Imported.SiteImport
embed_templates("../templates/site/settings_search_console.html")
embed_templates("../templates/site/settings_google_import.html")
end

43
lib/plausible_web/controllers/site_controller.ex
View File

@ -237,8 +237,10 @@ defmodule PlausibleWeb.SiteController do
Plausible.Google.API.fetch_verified_properties(site.google_auth)
end
legacy_import = Plausible.Imported.get_legacy_import(site)
imported_pageviews =
if site.imported_data do
if legacy_import do
Plausible.Stats.Clickhouse.imported_pageview_count(site)
else
0
@ -249,6 +251,7 @@ defmodule PlausibleWeb.SiteController do
conn
|> render("settings_integrations.html",
site: site,
legacy_import: legacy_import,
imported_pageviews: imported_pageviews,
has_plugins_tokens?: has_plugins_tokens?,
search_console_domains: search_console_domains,
@ -646,34 +649,18 @@ defmodule PlausibleWeb.SiteController do
def forget_import(conn, %{"import_id" => import_id}) do
site = conn.assigns.site
cond do
import_id == "0" ->
Plausible.Purge.delete_imported_stats!(site, 0)
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
site_import = Plausible.Imported.get_import(site, import_id) ->
Oban.cancel_all_jobs(
from(j in Oban.Job,
where:
j.queue == "analytics_imports" and
fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id
)
if site_import = Plausible.Imported.get_import(site, import_id) do
Oban.cancel_all_jobs(
from(j in Oban.Job,
where:
j.queue == "analytics_imports" and
fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id
)
)
Plausible.Purge.delete_imported_stats!(site_import)
Plausible.Purge.delete_imported_stats!(site_import)
Plausible.Repo.delete!(site_import)
if site_import.legacy do
Plausible.Purge.delete_imported_stats!(site, 0)
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
end
Plausible.Repo.delete!(site_import)
end
conn
@ -701,10 +688,6 @@ defmodule PlausibleWeb.SiteController do
Plausible.Purge.delete_imported_stats!(site)
Plausible.Imported.delete_imports_for_site(site)
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
end
conn

12
lib/plausible_web/live/imports_exports_settings.ex
View File

@ -137,7 +137,7 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
<%= Plausible.Imported.SiteImport.label(entry.site_import) %>
<span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">
(<%= PlausibleWeb.StatsView.large_number_format(
Map.get(@pageview_counts, entry.site_import.id, 0)
pageview_count(entry.site_import, @pageview_counts)
) %> page views)
</span>
</p>
@ -183,6 +183,16 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
{:noreply, assign(socket, site_imports: site_imports, pageview_counts: pageview_counts)}
end
defp pageview_count(site_import, pageview_counts) do
count = Map.get(pageview_counts, site_import.id, 0)
if site_import.legacy do
count + Map.get(pageview_counts, 0, 0)
else
count
end
end
defp update_imports(site_imports, import_id, status_str) do
Enum.map_reduce(site_imports, false, fn
%{site_import: %{id: ^import_id}} = entry, _changed? ->

18
lib/plausible_web/templates/site/settings_google_import.html.heex
View File

@ -11,11 +11,11 @@
<%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %>
<%= cond do %>
<% @site.imported_data && @site.imported_data.status == "importing" -> %>
<% @legacy_import && @legacy_import.status in [SiteImport.pending(), SiteImport.importing()] -> %>
<li class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
Import from <%= @site.imported_data.source %>
Import from Google Analytics
<svg
class="animate-spin -mr-1 ml-1 h-4 w-4 inline text-indigo-600"
xmlns="http://www.w3.org/2000/svg"
@ -40,8 +40,8 @@
</svg>
</p>
<p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
@site.imported_data.end_date
From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
@legacy_import.end_date
) %>
</p>
</div>
@ -52,11 +52,11 @@
"inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150"
) %>
</li>
<% @site.imported_data && @site.imported_data.status == "ok" -> %>
<% @legacy_import && @legacy_import.status == SiteImport.completed() -> %>
<li class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
Import from <%= @site.imported_data.source %>
Import from Google Analytics
<svg
class="h-4 w-4 inline ml-1 -mt-1 text-green-600"
xmlns="http://www.w3.org/2000/svg"
@ -69,8 +69,8 @@
</svg>
</p>
<p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
@site.imported_data.end_date
From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
@legacy_import.end_date
) %>
</p>
</div>
@ -85,7 +85,7 @@
) %>
</li>
<% true -> %>
<%= if @site.imported_data && @site.imported_data.status == "error" do %>
<%= if @legacy_import && @legacy_import.status == SiteImport.failed() do %>
<div class="text-sm mt-2 text-gray-900 dark:text-gray-100">
Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.
</div>

1
lib/plausible_web/templates/site/settings_integrations.html.heex
View File

@ -7,6 +7,7 @@
<PlausibleWeb.Components.Settings.settings_google_import
site={@site}
imported_pageviews={@imported_pageviews}
legacy_import={@legacy_import}
/>
<% end %>

4
lib/plausible_web/views/site_view.ex
View File

@ -2,6 +2,10 @@ defmodule PlausibleWeb.SiteView do
use PlausibleWeb, :view
use Plausible
alias Plausible.Imported.SiteImport
require Plausible.Imported.SiteImport
def plausible_url do
PlausibleWeb.Endpoint.url()
end

45
priv/repo/seeds.exs
View File

@ -256,51 +256,6 @@ native_stats_range
end)
|> Plausible.TestUtils.populate_stats()
site =
site
|> Plausible.Site.start_import(
legacy_imported_stats_range.first,
legacy_imported_stats_range.last,
"Google Analytics"
)
|> Plausible.Repo.update!()
legacy_imported_stats_range
|> Enum.flat_map(fn date ->
Enum.flat_map(0..Enum.random(1..500), fn _ ->
[
Plausible.Factory.build(:imported_visitors,
date: date,
pageviews: Enum.random(1..20),
visitors: Enum.random(1..20),
bounces: Enum.random(1..20),
visits: Enum.random(1..200),
visit_duration: Enum.random(1000..10000)
),
Plausible.Factory.build(:imported_sources,
date: date,
source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
visitors: Enum.random(1..20),
visits: Enum.random(1..200),
bounces: Enum.random(1..20),
visit_duration: Enum.random(1000..10000)
),
Plausible.Factory.build(:imported_pages,
date: date,
visitors: Enum.random(1..20),
pageviews: Enum.random(1..20),
exits: Enum.random(1..20),
time_on_page: Enum.random(1000..10000)
)
]
end)
end)
|> then(&Plausible.TestUtils.populate_stats(site, &1))
site
|> Plausible.Site.import_success()
|> Plausible.Repo.update!()
site_import =
site
|> Plausible.Imported.SiteImport.create_changeset(user, %{

33
test/plausible/data_migration/site_imports_test.exs
View File

@ -6,7 +6,6 @@ defmodule Plausible.DataMigration.SiteImportsTest do
alias Plausible.DataMigration.SiteImports
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
describe "run/1" do
test "runs for empty dataset" do
@ -30,7 +29,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "adds site import entry when it's missing and adjusts end date" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@ -54,7 +53,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "runs in dry mode without making any persistent changes" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@ -71,15 +70,14 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site = Repo.reload!(site)
assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)
assert id == 0
assert [] = Imported.list_all_imports(site)
assert site.imported_data.end_date == ~D[2021-01-08]
end
test "does not set end date to latter than the current one" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@ -103,7 +101,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "removes site import when there are no stats" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
|> Repo.update!()
assert capture_io(fn ->
@ -118,7 +116,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "leaves site and imports unchanged if everything fits" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
existing_import =
@ -150,7 +148,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "only considers sites with completed site imports or 'ok' imported data" do
site1 =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
existing_import1 =
@ -164,7 +162,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site2 =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
existing_import2 =
@ -178,12 +176,12 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site3 =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
site4 =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
_existing_import3 =
@ -272,4 +270,15 @@ defmodule Plausible.DataMigration.SiteImportsTest do
end
end
end
defp start_import(site, start_date, end_date, imported_source, status) do
change(site,
imported_data: %{
start_date: start_date,
end_date: end_date,
source: imported_source,
status: status
}
)
end
end

2
test/plausible/imported/csv_importer_test.exs
View File

@ -70,8 +70,6 @@ defmodule Plausible.Imported.CSVImporterTest do
}
] = Plausible.Imported.list_all_imports(site)
assert %{imported_data: nil} = Repo.reload!(site)
assert CSVImporter.parse_args(args) == [
uploads: uploads,
storage: on_full_build(do: "s3", else: "local")

11
test/plausible/imported/universal_analytics_test.exs
View File

@ -50,15 +50,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
}
] = Plausible.Imported.list_all_imports(site)
assert %{
imported_data: %{
source: "Google Analytics",
start_date: ~D[2023-10-01],
end_date: ~D[2024-01-02],
status: "importing"
}
} = Repo.reload!(site)
assert opts = [_ | _] = UniversalAnalytics.parse_args(args)
assert opts[:view_id] == "123"
@ -91,8 +82,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
legacy: false
}
] = Plausible.Imported.list_all_imports(site)
assert %{imported_data: nil} = Repo.reload!(site)
end
end

50
test/plausible/imported_test.exs
View File

@ -30,34 +30,6 @@ defmodule Plausible.ImportedTest do
assert import3.id in ids
assert import4.id in ids
end
test "returns one legacy import when present with respective site import entry" do
site = insert(:site)
{:ok, opts} = add_imported_data(%{site: site})
site = Map.new(opts).site
site_import = insert(:site_import, site: site, legacy: true)
site_import_id = site_import.id
assert [%{id: ^site_import_id}] = Imported.list_all_imports(site)
end
test "returns legacy import without respective site import entry" do
site = insert(:site)
{:ok, opts} = add_imported_data(%{site: site})
site = Map.new(opts).site
imported_start_date = site.imported_data.start_date
imported_end_date = site.imported_data.end_date
assert [
%{
id: 0,
source: :universal_analytics,
start_date: ^imported_start_date,
end_date: ^imported_end_date,
status: :completed
}
] = Imported.list_all_imports(site)
end
end
describe "get_imports_date_range/1" do
@ -68,15 +40,7 @@ defmodule Plausible.ImportedTest do
end
test "returns empty when only incomplete or failed imports are present" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"error"
)
|> Repo.update!()
site = insert(:site)
_import1 = insert(:site_import, site: site, status: :pending)
_import2 = insert(:site_import, site: site, status: :importing)
@ -87,15 +51,7 @@ defmodule Plausible.ImportedTest do
end
test "returns start and end dates considering all imports" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"ok"
)
|> Repo.update!()
site = insert(:site)
_import1 =
insert(:site_import,
@ -114,7 +70,7 @@ defmodule Plausible.ImportedTest do
status: :completed
)
assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =
assert %{start_date: ~D[2020-04-02], end_date: ~D[2024-01-08]} =
Imported.get_imports_date_range(site)
end
end

2
test/plausible/site/sites_test.exs
View File

@ -90,8 +90,6 @@ defmodule Plausible.SitesTest do
test "ignores imported stats" do
site = insert(:site)
insert(:site_import, site: site)
{:ok, opts} = add_imported_data(%{site: site})
site = Map.new(opts).site
assert Sites.native_stats_start_date(site) == nil
end

2
test/plausible/stats/clickhouse_test.exs
View File

@ -258,8 +258,6 @@ defmodule Plausible.Stats.ClickhouseTest do
describe "imported_pageview_counts/1" do
test "gets pageview counts for each of sites' imports" do
site = insert(:site)
{:ok, opts} = add_imported_data(%{site: site})
site = Map.new(opts).site
import1 = insert(:site_import, site: site)
import2 = insert(:site_import, site: site)

2
test/plausible/stats/comparisons_test.exs
View File

@ -203,7 +203,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "include_imported" do
setup [:create_user, :create_new_site, :add_imported_data]
setup [:create_user, :create_new_site, :create_site_import]
test "defaults to source_query.include_imported", %{site: site} do
query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})

34
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -473,10 +473,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
describe "with imported data" do
setup :add_imported_data
setup :create_site_import
test "does not count imported stats unless specified", %{conn: conn, site: site} do
populate_stats(site, [
test "does not count imported stats unless specified", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
])
@ -501,8 +505,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do
populate_stats(site, [
test "counts imported stats when comparing with previous period", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:imported_visitors,
visits: 2,
bounces: 1,
@ -540,8 +548,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
test "ignores imported data when filters are applied", %{conn: conn, site: site} do
populate_stats(site, [
test "ignores imported data when filters are applied", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:imported_sources, date: ~D[2023-01-01]),
build(:pageview,
@ -566,8 +578,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
test "events metric with imported data is disallowed", %{conn: conn, site: site} do
populate_stats(site, [
test "events metric with imported data is disallowed", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01])
])

52
test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View File

@ -330,12 +330,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
@ -520,12 +523,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
@ -842,12 +848,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",
%{conn: conn, site: site} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
@ -2378,12 +2387,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "metrics" do
test "returns time_on_page with imported data", %{conn: conn, site: site} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:imported_pages, page: "/A", time_on_page: 40, date: ~D[2021-01-01]),
build(:imported_pages, page: "/A", time_on_page: 110, date: ~D[2021-01-01]),
build(:imported_pages, page: "/B", time_on_page: 499, date: ~D[2021-01-01]),

22
test/plausible_web/controllers/api/stats_controller/browsers_test.exs
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/browsers" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_site_import]
test "returns top browsers by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
@ -109,8 +109,12 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
]
end
test "returns top browsers including imported data", %{conn: conn, site: site} do
populate_stats(site, [
test "returns top browsers including imported data", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:pageview, browser: "Chrome"),
build(:imported_browsers, browser: "Chrome"),
build(:imported_browsers, browser: "Firefox"),
@ -133,9 +137,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
test "skips breakdown when visitors=0 (possibly due to 'Enable Users Metric' in GA)", %{
conn: conn,
site: site
site: site,
site_import: site_import
} do
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:imported_browsers, browser: "Chrome", visitors: 0, visits: 14),
build(:imported_browsers, browser: "Firefox", visitors: 0, visits: 14),
build(:imported_browsers,
@ -169,9 +174,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
test "select empty imported_browsers as (not set), merging with the native (not set)", %{
conn: conn,
site: site
site: site,
site_import: site_import
} do
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:pageview, user_id: 123),
build(:imported_browsers, visitors: 1),
build(:imported_visitors, visitors: 1)
@ -186,7 +192,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
end
describe "GET /api/stats/:domain/browser-versions" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns correct conversion_rate when browser_version clashes across browsers", %{
conn: conn,

2
test/plausible_web/controllers/api/stats_controller/cities_test.exs
View File

@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do
])
end
setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]
test "returns top cities by new visitors", %{conn: conn, site: site} do
conn = get(conn, "/api/stats/#{site.domain}/cities?period=day")

12
test/plausible_web/controllers/api/stats_controller/conversions_test.exs
View File

@ -631,12 +631,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
conn: conn,
site: site
} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
site_import =
insert(:site_import,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, [
populate_stats(site, site_import.id, [
build(:pageview, pathname: "/"),
build(:pageview, pathname: "/another"),
build(:pageview, pathname: "/blog/post-1"),

2
test/plausible_web/controllers/api/stats_controller/countries_test.exs
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/countries" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top countries by new visitors", %{conn: conn, site: site} do
populate_stats(site, [

2
test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/custom-prop-values/:prop_key" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns breakdown by a custom property", %{conn: conn, site: site} do
prop_key = "parim_s6ber"

51
test/plausible_web/controllers/api/stats_controller/imported_test.exs
View File

@ -10,43 +10,30 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
|> then(&Plausible.Imported.Buffer.insert_all(table_name, &1))
end
for import_type <- [:legacy, :new_and_legacy, :new] do
for import_type <- [:new_and_legacy, :new] do
describe "Parse and import third party data fetched from Google Analytics as #{import_type} import" do
if import_type == :new do
setup [:create_user, :log_in, :create_new_site]
else
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
end
setup [:create_user, :log_in, :create_new_site]
setup %{user: user, site: site} do
if unquote(import_type) in [:new, :new_and_legacy] do
import_params =
if unquote(import_type) == :new_and_legacy do
site.imported_data
|> Map.from_struct()
|> Map.put(:source, :universal_analytics)
else
%{
source: :universal_analytics,
start_date: ~D[2005-01-01],
end_date: Timex.today()
}
end
import_params =
%{
source: :universal_analytics,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
legacy: unquote(import_type) == :new_and_legacy
}
site_import =
site
|> Plausible.Imported.SiteImport.create_changeset(
user,
import_params
)
|> Plausible.Repo.insert!()
|> Plausible.Imported.SiteImport.complete_changeset()
|> Plausible.Repo.update!()
site_import =
site
|> Plausible.Imported.SiteImport.create_changeset(
user,
import_params
)
|> Plausible.Repo.insert!()
|> Plausible.Imported.SiteImport.complete_changeset()
|> Plausible.Repo.update!()
{:ok, %{import_id: site_import.id}}
else
{:ok, %{import_id: 0}}
end
{:ok, %{import_id: site_import.id}}
end
test "Visitors data imported from Google Analytics", %{

36
test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
View File

@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@user_id 123
describe "GET /api/stats/main-graph - plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} do
populate_stats(site, [
@ -368,7 +368,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - pageviews plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays pageviews for a month", %{conn: conn, site: site} do
populate_stats(site, [
@ -556,7 +556,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - bounce_rate plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays bounce_rate for a month", %{conn: conn, site: site} do
populate_stats(site, [
@ -620,7 +620,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - visit_duration plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays visit_duration for a month", %{conn: conn, site: site} do
populate_stats(site, [
@ -915,7 +915,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - comparisons" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns past month stats when period=30d and comparison=previous_period", %{
conn: conn,
@ -1007,17 +1007,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
site
|> Ecto.Changeset.change(
imported_data: %{
start_date: ~D[2005-01-01],
end_date: ~D[2020-01-31],
source: "Google Analytics",
status: "ok"
}
)
|> Plausible.Repo.update!()
conn =
get(
conn,
@ -1064,17 +1053,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
site
|> Ecto.Changeset.change(
imported_data: %{
start_date: ~D[2005-01-01],
end_date: ~D[2020-01-31],
source: "Google Analytics",
status: "ok"
}
)
|> Plausible.Repo.update!()
conn =
get(
conn,
@ -1125,7 +1103,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@tag :full_build_only
describe "GET /api/stats/main-graph - total_revenue plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "plots total_revenue for a month", %{conn: conn, site: site} do
insert(:goal, site: site, event_name: "Payment", currency: "USD")
@ -1205,7 +1183,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@tag :full_build_only
describe "GET /api/stats/main-graph - average_revenue plot" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "plots total_revenue for a month", %{conn: conn, site: site} do
insert(:goal, site: site, event_name: "Payment", currency: "USD")

4
test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/operating-systems" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns operating systems by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
@ -211,7 +211,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
end
describe "GET /api/stats/:domain/operating-system-versions" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top OS versions by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [

6
test/plausible_web/controllers/api/stats_controller/pages_test.exs
View File

@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
@user_id 123
describe "GET /api/stats/:domain/pages" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
@ -1202,7 +1202,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
end
describe "GET /api/stats/:domain/entry-pages" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top entry pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
@ -1555,7 +1555,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
end
describe "GET /api/stats/:domain/exit-pages" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top exit pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [

2
test/plausible_web/controllers/api/stats_controller/regions_test.exs
View File

@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do
])
end
setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]
test "returns top cities by new visitors", %{conn: conn, site: site} do
conn = get(conn, "/api/stats/#{site.domain}/regions?period=day")

2
test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/screen-sizes" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns screen sizes by new visitors", %{conn: conn, site: site} do
populate_stats(site, [

12
test/plausible_web/controllers/api/stats_controller/sources_test.exs
View File

@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
@user_id 123
describe "GET /api/stats/:domain/sources" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top sources by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@ -544,7 +544,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_mediums" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@ -696,7 +696,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_campaigns" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@ -856,7 +856,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_sources" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_sources by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_terms" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_terms by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@ -1064,7 +1064,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_contents" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_contents by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [

26
test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
View File

@ -463,7 +463,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - with imported data" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns divisible metrics as 0 when no stats exist", %{
site: site,
@ -1267,7 +1267,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - with comparisons" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "does not return comparisons by default", %{site: site, conn: conn} do
populate_stats(site, [
@ -1352,17 +1352,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
site
|> Ecto.Changeset.change(
imported_data: %{
start_date: ~D[2005-01-01],
end_date: ~D[2020-01-31],
source: "Google Analytics",
status: "ok"
}
)
|> Plausible.Repo.update!()
conn =
get(
conn,
@ -1393,17 +1382,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
site
|> Ecto.Changeset.change(
imported_data: %{
start_date: ~D[2005-01-01],
end_date: ~D[2020-01-31],
source: "Google Analytics",
status: "ok"
}
)
|> Plausible.Repo.update!()
conn =
get(
conn,

42
test/plausible_web/controllers/site_controller_test.exs
View File

@ -655,9 +655,6 @@ defmodule PlausibleWeb.SiteControllerTest do
end
test "renders imports in import list", %{conn: conn, site: site} do
{:ok, opts} = add_imported_data(%{site: site})
site = Map.new(opts).site
_site_import1 = insert(:site_import, site: site, status: SiteImport.pending())
_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
@ -675,7 +672,7 @@ defmodule PlausibleWeb.SiteControllerTest do
resp = html_response(conn, 200)
buttons = find(resp, ~s|button[data-method="delete"]|)
assert length(buttons) == 5
assert length(buttons) == 4
assert resp =~ "Google Analytics (123456)"
assert resp =~ "(98 page views)"
@ -693,6 +690,20 @@ defmodule PlausibleWeb.SiteControllerTest do
"Maximum of #{Plausible.Imported.max_complete_imports()} imports is reached."
end
test "considers older legacy imports when showing pageview count", %{conn: conn, site: site} do
_site_import =
insert(:site_import, site: site, legacy: true, status: SiteImport.completed())
populate_stats(site, [
build(:imported_visitors, pageviews: 77),
build(:imported_visitors, pageviews: 21)
])
conn = get(conn, "/#{site.domain}/settings/imports-exports")
assert html_response(conn, 200) =~ "(98 page views)"
end
test "disables import buttons when there's import in progress", %{conn: conn, site: site} do
_site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
@ -1312,7 +1323,7 @@ defmodule PlausibleWeb.SiteControllerTest do
end
describe "DELETE /:website/settings/:forget_import/:import_id" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "removes site import, associated data and cancels oban job for a particular import", %{
conn: conn,
@ -1338,7 +1349,11 @@ defmodule PlausibleWeb.SiteControllerTest do
build(:imported_visitors, pageviews: 10)
])
assert [%{id: ^import_id}, %{id: 0}] = Plausible.Imported.list_all_imports(site)
imports = Plausible.Imported.list_all_imports(site)
assert Enum.find(imports, &(&1.id == import_id))
site = Plausible.Imported.load_import_data(site)
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
@ -1355,9 +1370,10 @@ defmodule PlausibleWeb.SiteControllerTest do
assert Repo.reload(job).state == "cancelled"
end
test "removes legacy site import along with associated data when instructed", %{
test "removes all legacy site import data when instructed", %{
conn: conn,
site: site
site: site,
site_import: legacy_site_import
} do
other_site_import = insert(:site_import, site: site)
@ -1375,7 +1391,7 @@ defmodule PlausibleWeb.SiteControllerTest do
{count == 22, count}
end)
delete(conn, "/#{site.domain}/settings/forget-import/0")
delete(conn, "/#{site.domain}/settings/forget-import/#{legacy_site_import.id}")
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
@ -1385,13 +1401,7 @@ defmodule PlausibleWeb.SiteControllerTest do
end
describe "DELETE /:website/settings/forget_imported" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "removes imported_data field from site", %{conn: conn, site: site} do
delete(conn, "/#{site.domain}/settings/forget-imported")
assert Repo.reload(site).imported_data == nil
end
setup [:create_user, :log_in, :create_new_site]
test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} do
Plausible.Imported.NoopImporter.new_import(

20
test/support/test_utils.ex
View File

@ -47,13 +47,21 @@ defmodule Plausible.TestUtils do
{:ok, site: site}
end
def add_imported_data(%{site: site}) do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Repo.update!()
def create_legacy_site_import(%{site: site}) do
create_site_import(%{site: site, create_legacy_import?: true})
end
{:ok, site: site}
def create_site_import(%{site: site} = opts) do
site_import =
Factory.insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics,
legacy: opts[:create_legacy_import?] == true
)
{:ok, site_import: site_import}
end
def create_new_site(%{user: user}) do