From 8fb4f3f88644b4a0029be61975cabe3f054fcdde Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 21 Apr 2022 19:22:38 +0300 Subject: [PATCH] Revert entry props --- lib/plausible/session/clickhouse_schema.ex | 4 +- lib/plausible/session/store.ex | 6 +- .../20220421161259_remove_entry_props.exs | 9 + test/plausible/session/store_test.exs | 4 +- test/plausible/site/admin_test.exs | 2 +- .../api/stats_controller/sources_test.exs | 324 +++++++++--------- 6 files changed, 179 insertions(+), 170 deletions(-) create mode 100644 priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exs diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index b66188a7b..348984319 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -18,8 +18,8 @@ defmodule Plausible.ClickhouseSession do field :events, :integer field :sign, :integer - field :"entry.meta.key", {:array, :string}, default: [] - field :"entry.meta.value", {:array, :string}, default: [] + # field :"entry.meta.key", {:array, :string}, default: [] + # field :"entry.meta.value", {:array, :string}, default: [] field :utm_medium, :string field :utm_source, :string diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index 9c8cfbccd..4a3f4f905 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -160,9 +160,9 @@ defmodule Plausible.Session.Store do browser: event.browser, browser_version: event.browser_version, timestamp: event.timestamp, - start: event.timestamp, - "entry.meta.key": Map.get(event, :"meta.key"), - "entry.meta.value": Map.get(event, :"meta.value") + start: event.timestamp + # "entry.meta.key": Map.get(event, :"meta.key"), + # "entry.meta.value": Map.get(event, :"meta.value") } end diff --git a/priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exs b/priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exs new file mode 100644 index 000000000..26456213a --- /dev/null +++ b/priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.RemoveEntryProps do + use Ecto.Migration + + def change do + alter table(:sessions) do + remove(:"entry.meta") + end + end +end diff --git a/test/plausible/session/store_test.exs b/test/plausible/session/store_test.exs index 78eed21df..6738e1454 100644 --- a/test/plausible/session/store_test.exs +++ b/test/plausible/session/store_test.exs @@ -61,8 +61,8 @@ defmodule Plausible.Session.StoreTest do assert session.browser_version == event.browser_version assert session.timestamp == event.timestamp assert session.start === event.timestamp - assert Map.get(session, :"entry.meta.key") == ["logged_in", "darkmode"] - assert Map.get(session, :"entry.meta.value") == ["true", "false"] + # assert Map.get(session, :"entry.meta.key") == ["logged_in", "darkmode"] + # assert Map.get(session, :"entry.meta.value") == ["true", "false"] end test "updates a session", %{store: store} do diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs index 1b2b6ed32..f5cd92538 100644 --- a/test/plausible/site/admin_test.exs +++ b/test/plausible/site/admin_test.exs @@ -66,7 +66,7 @@ defmodule Plausible.SiteAdminTest do actual = SiteAdmin.session_transfer_query("from.com", "to.com") expected = - "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry.meta.key, entry.meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry.meta.key, entry.meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" assert actual == expected end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 0ece9bca9..e44fbe8b5 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -30,183 +30,183 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end - test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", - user_id: 123, - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - "meta.key": ["author"], - "meta.value": ["John Doe"], - user_id: 123, - timestamp: ~N[2021-01-01 00:01:00] - ), - build(:pageview, - referrer_source: "Google", - referrer: "google.com", - "meta.key": ["author"], - "meta.value": ["John Doe"], - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", - timestamp: ~N[2021-01-01 00:00:00] - ) - ]) + # test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, + # referrer_source: "DuckDuckGo", + # referrer: "duckduckgo.com", + # user_id: 123, + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # "meta.key": ["author"], + # "meta.value": ["John Doe"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # referrer_source: "Google", + # referrer: "google.com", + # "meta.key": ["author"], + # "meta.value": ["John Doe"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # referrer_source: "Facebook", + # referrer: "facebook.com", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) - filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + # filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) - conn = - get( - conn, - "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - ) + # conn = + # get( + # conn, + # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + # ) - assert json_response(conn, 200) == [ - %{"name" => "Google", "visitors" => 1} - ] - end + # assert json_response(conn, 200) == [ + # %{"name" => "Google", "visitors" => 1} + # ] + # end - test "returns top sources with :is_not filter on custom pageview props", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", - "meta.key": ["author"], - "meta.value": ["John Doe"], - user_id: 123, - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - "meta.key": ["author"], - "meta.value": ["other"], - user_id: 123, - timestamp: ~N[2021-01-01 00:01:00] - ), - build(:pageview, - referrer_source: "Google", - referrer: "google.com", - "meta.key": ["author"], - "meta.value": ["other"], - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", - timestamp: ~N[2021-01-01 00:00:00] - ) - ]) + # test "returns top sources with :is_not filter on custom pageview props", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, + # referrer_source: "DuckDuckGo", + # referrer: "duckduckgo.com", + # "meta.key": ["author"], + # "meta.value": ["John Doe"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # "meta.key": ["author"], + # "meta.value": ["other"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # referrer_source: "Google", + # referrer: "google.com", + # "meta.key": ["author"], + # "meta.value": ["other"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # referrer_source: "Facebook", + # referrer: "facebook.com", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) - filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + # filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) - conn = - get( - conn, - "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - ) + # conn = + # get( + # conn, + # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + # ) - assert json_response(conn, 200) == [ - %{"name" => "Facebook", "visitors" => 1}, - %{"name" => "Google", "visitors" => 1} - ] - end + # assert json_response(conn, 200) == [ + # %{"name" => "Facebook", "visitors" => 1}, + # %{"name" => "Google", "visitors" => 1} + # ] + # end - test "returns top sources with :is (none) filter on custom pageview props", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", - "meta.key": ["author"], - "meta.value": ["John Doe"], - user_id: 123, - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - user_id: 123, - timestamp: ~N[2021-01-01 00:01:00] - ), - build(:pageview, - referrer_source: "Google", - referrer: "google.com", - "meta.key": ["author"], - "meta.value": ["other"], - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", - timestamp: ~N[2021-01-01 00:00:00] - ) - ]) + # test "returns top sources with :is (none) filter on custom pageview props", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, + # referrer_source: "DuckDuckGo", + # referrer: "duckduckgo.com", + # "meta.key": ["author"], + # "meta.value": ["John Doe"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # user_id: 123, + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # referrer_source: "Google", + # referrer: "google.com", + # "meta.key": ["author"], + # "meta.value": ["other"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # referrer_source: "Facebook", + # referrer: "facebook.com", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) - filters = Jason.encode!(%{props: %{"author" => "(none)"}}) + # filters = Jason.encode!(%{props: %{"author" => "(none)"}}) - conn = - get( - conn, - "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - ) + # conn = + # get( + # conn, + # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + # ) - assert json_response(conn, 200) == [ - %{"name" => "Facebook", "visitors" => 1} - ] - end + # assert json_response(conn, 200) == [ + # %{"name" => "Facebook", "visitors" => 1} + # ] + # end - test "returns top sources with :is_not (none) filter on custom pageview props", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", - "meta.key": ["logged_in"], - "meta.value": ["true"], - user_id: 123, - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - "meta.key": ["author"], - "meta.value": ["other"], - user_id: 123, - timestamp: ~N[2021-01-01 00:01:00] - ), - build(:pageview, - referrer_source: "Google", - referrer: "google.com", - "meta.key": ["author"], - "meta.value": ["other"], - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", - timestamp: ~N[2021-01-01 00:00:00] - ) - ]) + # test "returns top sources with :is_not (none) filter on custom pageview props", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, + # referrer_source: "DuckDuckGo", + # referrer: "duckduckgo.com", + # "meta.key": ["logged_in"], + # "meta.value": ["true"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # "meta.key": ["author"], + # "meta.value": ["other"], + # user_id: 123, + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # referrer_source: "Google", + # referrer: "google.com", + # "meta.key": ["author"], + # "meta.value": ["other"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # referrer_source: "Facebook", + # referrer: "facebook.com", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) - filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) + # filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) - conn = - get( - conn, - "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - ) + # conn = + # get( + # conn, + # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + # ) - assert json_response(conn, 200) == [ - %{"name" => "Google", "visitors" => 1} - ] - end + # assert json_response(conn, 200) == [ + # %{"name" => "Google", "visitors" => 1} + # ] + # end test "returns top sources with imported data", %{conn: conn, site: site} do populate_stats(site, [