Start putting together means to export all aggregate data (#1387)

* Export all dashboard data in zip

This packages all data currently visible in the dashboard into
individual CSVs and downloads them together in a zip.

* Also export conversions in zip of CSVs

* Update export test with zip file response

* Add zip file download to changelog
This commit is contained in:
Matt Colligan 2021-10-26 14:54:50 +01:00 committed by GitHub
parent 7622621a1d
commit 8138a6e3a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 253 additions and 35 deletions

View File

@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
- Add Conversion Rate to Top Sources, Top Pages Devices, Countries when filtered by a goal plausible/analytics#1299
- Add list view for countries report in dashboard plausible/analytics#1381
- Add ability to view more than 100 custom goal properties plausible/analytics#1353
- Data exported via the download button will contain CSV data for all visible graps in a zip file.
### Fixed
- Fix weekly report time range plausible/analytics#951

View File

@ -303,7 +303,7 @@ class LineGraph extends React.Component {
downloadLink() {
if (this.props.query.period !== 'realtime') {
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/visitors.csv${api.serializeQuery(this.props.query)}`
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}`
return (
<a href={endpoint} download>

View File

@ -216,7 +216,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "source", "visit:source")
|> transform_keys(%{"source" => "name", "visitors" => "count"})
json(conn, res)
if params["csv"] do
res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
else
json(conn, res)
end
end
def utm_mediums(conn, params) do
@ -235,7 +239,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium")
|> transform_keys(%{"utm_medium" => "name", "visitors" => "count"})
json(conn, res)
if params["csv"] do
res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
else
json(conn, res)
end
end
def utm_campaigns(conn, params) do
@ -254,7 +262,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign")
|> transform_keys(%{"utm_campaign" => "name", "visitors" => "count"})
json(conn, res)
if params["csv"] do
res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
else
json(conn, res)
end
end
def utm_sources(conn, params) do
@ -273,7 +285,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source")
|> transform_keys(%{"utm_source" => "name", "visitors" => "count"})
json(conn, res)
if params["csv"] do
res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
else
json(conn, res)
end
end
def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do
@ -344,7 +360,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "page", "event:page")
|> transform_keys(%{"page" => "name", "visitors" => "count"})
json(conn, pages)
if params["csv"] do
pages |> to_csv(["name", "count", "bounce_rate", "time_on_page"])
else
json(conn, pages)
end
end
def entry_pages(conn, params) do
@ -358,7 +378,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page")
|> transform_keys(%{"entry_page" => "name", "visits" => "entries", "visitors" => "count"})
json(conn, entry_pages)
if params["csv"] do
entry_pages |> to_csv(["name", "count", "entries", "visit_duration"])
else
json(conn, entry_pages)
end
end
def exit_pages(conn, params) do
@ -398,7 +422,11 @@ defmodule PlausibleWeb.Api.StatsController do
Map.put(exit_page, "exit_rate", exit_rate)
end)
json(conn, exit_pages)
if params["csv"] do
exit_pages |> to_csv(["name", "count", "exits", "exit_rate"])
else
json(conn, exit_pages)
end
end
def countries(conn, params) do
@ -416,7 +444,11 @@ defmodule PlausibleWeb.Api.StatsController do
end)
|> add_percentages
json(conn, countries)
if params["csv"] do
countries |> to_csv(["name", "count"])
else
json(conn, countries)
end
end
def browsers(conn, params) do
@ -430,7 +462,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"browser" => "name", "visitors" => "count"})
|> add_percentages
json(conn, browsers)
if params["csv"] do
browsers |> to_csv(["name", "count"])
else
json(conn, browsers)
end
end
def browser_versions(conn, params) do
@ -458,7 +494,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"os" => "name", "visitors" => "count"})
|> add_percentages
json(conn, systems)
if params["csv"] do
systems |> to_csv(["name", "count"])
else
json(conn, systems)
end
end
def operating_system_versions(conn, params) do
@ -486,7 +526,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"device" => "name", "visitors" => "count"})
|> add_percentages
json(conn, sizes)
if params["csv"] do
sizes |> to_csv(["name", "count"])
else
json(conn, sizes)
end
end
defp calculate_cr(unique_visitors, converted_visitors) do
@ -526,7 +570,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["count"]))
end)
json(conn, conversions)
if params["csv"] do
conversions |> to_csv(["name", "count", "total_count"])
else
json(conn, conversions)
end
end
def prop_breakdown(conn, params) do
@ -634,4 +682,12 @@ defmodule PlausibleWeb.Api.StatsController do
list
end
end
defp to_csv(list, headers) do
list
|> Enum.map(fn row -> Enum.map(headers, &row[&1]) end)
|> (fn res -> [headers | res] end).()
|> CSV.encode()
|> Enum.join()
end
end

View File

@ -1,6 +1,7 @@
defmodule PlausibleWeb.StatsController do
use PlausibleWeb, :controller
use Plausible.Repo
alias PlausibleWeb.Api
alias Plausible.Stats.Query
plug PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export]
@ -37,9 +38,9 @@ defmodule PlausibleWeb.StatsController do
end
end
def csv_export(conn, %{"domain" => domain}) do
def csv_export(conn, params) do
site = conn.assigns[:site]
query = Query.from(site.timezone, conn.params)
query = Query.from(site.timezone, params)
metrics =
if query.filters["event:name"] do
@ -52,20 +53,40 @@ defmodule PlausibleWeb.StatsController do
headers = ["date" | metrics]
csv_content =
visitors =
Enum.map(graph, fn row -> Enum.map(headers, &row[&1]) end)
|> (fn data -> [headers | data] end).()
|> CSV.encode()
|> Enum.into([])
|> Enum.join()
filename =
"Plausible export #{domain} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.csv"
"Plausible export #{params["domain"]} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.zip"
params = Map.put(params, "limit", "1000")
params = Map.put(params, "csv", "True")
csvs = [
{'visitors.csv', visitors},
{'sources.csv', Api.StatsController.sources(conn, params)},
{'utm_mediums.csv', Api.StatsController.utm_mediums(conn, params)},
{'utm_sources.csv', Api.StatsController.utm_sources(conn, params)},
{'utm_campaigns.csv', Api.StatsController.utm_campaigns(conn, params)},
{'pages.csv', Api.StatsController.pages(conn, params)},
{'entry_pages.csv', Api.StatsController.entry_pages(conn, params)},
{'exit_pages.csv', Api.StatsController.exit_pages(conn, params)},
{'countries.csv', Api.StatsController.countries(conn, params)},
{'browsers.csv', Api.StatsController.browsers(conn, params)},
{'operating_systems.csv', Api.StatsController.operating_systems(conn, params)},
{'devices.csv', Api.StatsController.screen_sizes(conn, params)},
{'conversions.csv', Api.StatsController.conversions(conn, params)}
]
{:ok, {_, zip_content}} = :zip.create(filename, csvs, [:memory])
conn
|> put_resp_content_type("text/csv")
|> put_resp_content_type("application/zip")
|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")
|> send_resp(200, csv_content)
|> send_resp(200, zip_content)
end
def shared_link(conn, %{"slug" => domain, "auth" => auth}) do

View File

@ -234,7 +234,7 @@ defmodule PlausibleWeb.Router do
delete "/:website", SiteController, :delete_site
delete "/:website/stats", SiteController, :reset_stats
get "/:domain/visitors.csv", StatsController, :csv_export
get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats
end
end

View File

@ -44,31 +44,171 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
describe "GET /:website/visitors.csv" do
setup [:create_user, :log_in, :create_site]
describe "GET /:website/export" do
setup [:create_user, :create_new_site, :log_in]
test "exports graph as csv", %{conn: conn, site: site} do
today = Timex.today() |> Timex.format!("{ISOdate}")
test "exports data in zipped csvs", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
country_code: "EE",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
referrer_source: "Google"
),
build(:pageview,
utm_campaign: "ads",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
)
])
conn = get(conn, "/" <> site.domain <> "/visitors.csv")
assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"
assert response(conn, 200) =~ "#{today},3,3,0,0"
conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")
assert conn.status == 200
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
assert_csv(
zip,
'visitors.csv',
"date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
)
assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(
zip,
'utm_campaigns.csv',
"name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
)
assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")
assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")
assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")
assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")
assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
end
end
describe "GET /:website/visitors.csv - via shared link" do
test "exports graph as csv", %{conn: conn} do
site = insert(:site, domain: "test-site.com")
describe "GET /:website/export - via shared link" do
test "exports data in zipped csvs", %{conn: conn} do
site = insert(:site, domain: "new-site.com")
link = insert(:shared_link, site: site)
today = Timex.today() |> Timex.format!("{ISOdate}")
populate_stats(site, [
build(:pageview,
country_code: "EE",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
referrer_source: "Google"
),
build(:pageview,
utm_campaign: "ads",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
)
])
conn = get(conn, "/" <> site.domain <> "/visitors.csv?auth=#{link.slug}")
assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"
assert response(conn, 200) =~ "#{today},3,3,0,0"
conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")
assert conn.status == 200
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
assert_csv(
zip,
'visitors.csv',
"date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
)
assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(
zip,
'utm_campaigns.csv',
"name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
)
assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")
assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")
assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")
assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")
assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")
assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
end
end
describe "GET /:website/export - for past 6 months" do
setup [:create_user, :create_new_site, :log_in]
test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
timestamp: relative_time(minutes: -1)
),
build(:pageview,
timestamp: relative_time(months: -1),
country_code: "EE",
browser: "ABrowserName"
),
build(:pageview,
timestamp: relative_time(months: -5),
utm_campaign: "ads",
country_code: "EE",
referrer_source: "Google",
browser: "ABrowserName"
)
])
conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")
assert conn.status == 200
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
assert_csv(
zip,
'visitors.csv',
"date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"
)
assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
assert_csv(
zip,
'utm_campaigns.csv',
"name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
)
assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,3,,\r\n")
assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,3,3,0\r\n")
assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,3,3,100.0\r\n")
assert_csv(zip, 'countries.csv', "name,count\r\nEST,2\r\n")
assert_csv(zip, 'browsers.csv', "name,count\r\nABrowserName,2\r\n,1\r\n")
assert_csv(zip, 'operating_systems.csv', "name,count\r\n,3\r\n")
assert_csv(zip, 'devices.csv', "name,count\r\n,3\r\n")
assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
end
end
defp assert_csv(zip, fileName, string) do
{_, contents} = List.keyfind(zip, fileName, 0)
assert to_string(contents) == string
end
describe "GET /share/:slug" do
test "prompts a password for a password-protected link", %{conn: conn} do
site = insert(:site)