diff --git a/lib/plausible/auth/api_key.ex b/lib/plausible/auth/api_key.ex new file mode 100644 index 000000000..71fcd111b --- /dev/null +++ b/lib/plausible/auth/api_key.ex @@ -0,0 +1,45 @@ +defmodule Plausible.Auth.ApiKey do + use Ecto.Schema + import Ecto.Changeset + + @required [:user_id, :key, :name] + schema "api_keys" do + field :name, :string + field :key, :string, virtual: true + field :key_hash, :string + field :key_prefix, :string + + belongs_to :user, Plausible.Auth.User + + timestamps() + end + + def changeset(schema, attrs \\ %{}) do + schema + |> cast(attrs, @required) + |> validate_required(@required) + |> process_key + end + + def do_hash(key) do + :crypto.hash(:sha256, [secret_key_base(), key]) + |> Base.encode16() + |> String.downcase() + end + + def process_key(%{errors: [], changes: changes} = changeset) do + prefix = binary_part(changes[:key], 0, 6) + + change(changeset, + key_hash: do_hash(changes[:key]), + key_prefix: prefix + ) + end + + def process_key(changeset), do: changeset + + defp secret_key_base() do + Application.get_env(:plausible, PlausibleWeb.Endpoint) + |> Keyword.fetch!(:secret_key_base) + end +end diff --git a/lib/plausible/auth/user.ex b/lib/plausible/auth/user.ex index 12740b15c..a243afa54 100644 --- a/lib/plausible/auth/user.ex +++ b/lib/plausible/auth/user.ex @@ -22,6 +22,7 @@ defmodule Plausible.Auth.User do has_many :site_memberships, Plausible.Site.Membership has_many :sites, through: [:site_memberships, :site] + has_many :api_keys, Plausible.Auth.ApiKey has_one :google_auth, Plausible.Site.GoogleAuth has_one :subscription, Plausible.Billing.Subscription diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index c848dd43f..ae48d0240 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -73,6 +73,28 @@ defmodule Plausible.Session.Store do {:reply, session_id, %{state | sessions: updated_sessions}} end + def reconcile_event(sessions, event) do + found_session = sessions[event.user_id] + active = is_active?(found_session, event) + + updated_sessions = + cond do + found_session && active -> + new_session = update_session(found_session, event) + Map.put(sessions, event.user_id, new_session) + + found_session && !active -> + new_session = new_session_from_event(event) + Map.put(sessions, event.user_id, new_session) + + true -> + new_session = new_session_from_event(event) + Map.put(sessions, event.user_id, new_session) + end + + updated_sessions + end + defp is_active?(session, event) do session && Timex.diff(event.timestamp, session.timestamp, :second) < session_length_seconds() end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index fcb76d475..3fff24722 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -23,9 +23,11 @@ defmodule Plausible.Stats.Clickhouse do end end - def calculate_plot(site, %Query{step_type: "month"} = query) do + def calculate_plot(site, %Query{interval: "month"} = query) do + n_steps = Timex.diff(query.date_range.last, query.date_range.first, :months) + steps = - Enum.map((query.steps - 1)..0, fn shift -> + Enum.map(n_steps..0, fn shift -> Timex.now(site.timezone) |> Timex.beginning_of_month() |> Timex.shift(months: -shift) @@ -54,7 +56,7 @@ defmodule Plausible.Stats.Clickhouse do {plot, labels, present_index} end - def calculate_plot(site, %Query{step_type: "date"} = query) do + def calculate_plot(site, %Query{interval: "date"} = query) do steps = Enum.into(query.date_range, []) groups = @@ -78,7 +80,7 @@ defmodule Plausible.Stats.Clickhouse do {plot, labels, present_index} end - def calculate_plot(site, %Query{step_type: "hour"} = query) do + def calculate_plot(site, %Query{interval: "hour"} = query) do steps = 0..23 groups = @@ -151,6 +153,13 @@ defmodule Plausible.Stats.Clickhouse do ) end + def total_pageviews(site, query) do + ClickhouseRepo.one( + from e in base_query(site, query), + select: fragment("count(*)") + ) + end + def total_events(site, query) do ClickhouseRepo.one( from e in base_query_w_sessions(site, query), diff --git a/lib/plausible/stats/mod.ex b/lib/plausible/stats/mod.ex new file mode 100644 index 000000000..07af7d7cc --- /dev/null +++ b/lib/plausible/stats/mod.ex @@ -0,0 +1,239 @@ +defmodule Plausible.Stats do + use Plausible.ClickhouseRepo + alias Plausible.Stats.Query + @no_ref "Direct / None" + + def timeseries(site, query) do + steps = buckets(query) + + groups = + from(e in base_event_query(site, query), + group_by: fragment("bucket"), + order_by: fragment("bucket") + ) + |> select_bucket(site, query) + |> ClickhouseRepo.all() + |> Enum.into(%{}) + + plot = Enum.map(steps, fn step -> groups[step] || 0 end) + labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end) + + {plot, labels} + end + + @event_metrics ["visitors", "pageviews"] + @session_metrics ["bounce_rate", "visit_duration"] + + def aggregate(site, query, metrics) do + event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) + event_task = Task.async(fn -> aggregate_events(site, query, event_metrics) end) + session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) + session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end) + + Map.merge( + Task.await(event_task), + Task.await(session_task) + ) + |> Enum.map(fn {metric, value} -> + {metric, %{value: value}} + end) + |> Enum.into(%{}) + end + + defp aggregate_events(_, _, []), do: %{} + + defp aggregate_events(site, query, metrics) do + q = from(e in base_event_query(site, query), select: %{}) + + Enum.reduce(metrics, q, &select_event_metric/2) + |> ClickhouseRepo.one() + end + + defp select_event_metric("pageviews", q) do + from(e in q, select_merge: %{pageviews: fragment("count(*)")}) + end + + defp select_event_metric("visitors", q) do + from(e in q, select_merge: %{visitors: fragment("uniq(?)", e.user_id)}) + end + + defp aggregate_sessions(_, _, []), do: %{} + + defp aggregate_sessions(site, query, metrics) do + q = from(e in query_sessions(site, query), select: %{}) + + Enum.reduce(metrics, q, &select_session_metric/2) + |> ClickhouseRepo.one() + end + + defp select_session_metric("bounce_rate", q) do + from(s in q, + select_merge: %{bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)")} + ) + end + + defp select_session_metric("visit_duration", q) do + from(s in q, select_merge: %{visit_duration: fragment("round(avg(duration * sign))")}) + end + + @session_props [ + "source", + "referrer", + "utm_medium", + "utm_source", + "utm_campaign", + "device", + "browser", + "browser_version", + "os", + "os_version", + "country" + ] + + defp base_event_query(site, query) do + events_q = query_events(site, query) + + if Enum.any?(@session_props, &query.filters["visit:" <> &1]) do + sessions_q = + from( + s in query_sessions(site, query), + select: %{session_id: s.session_id} + ) + + from( + e in events_q, + join: sq in subquery(sessions_q), + on: e.session_id == sq.session_id + ) + else + events_q + end + end + + defp query_events(site, query) do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + q = + from(e in "events", + where: e.domain == ^site.domain, + where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime + ) + + q = + if query.filters["event:page"] do + page = query.filters["event:page"] + from(e in q, where: e.pathname == ^page) + else + q + end + + if query.filters["props"] do + [{key, val}] = query.filters["props"] |> Enum.into([]) + + if val == "(none)" do + from( + e in q, + where: fragment("not has(meta.key, ?)", ^key) + ) + else + from( + e in q, + inner_lateral_join: meta in fragment("meta as m"), + as: :meta, + where: meta.key == ^key and meta.value == ^val + ) + end + else + q + end + end + + defp query_sessions(site, query) do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + sessions_q = + from(s in "sessions", + where: s.domain == ^site.domain, + where: s.timestamp >= ^first_datetime and s.start < ^last_datetime + ) + + sessions_q = + if query.filters["event:page"] do + page = query.filters["event:page"] + from(e in sessions_q, where: e.entry_page == ^page) + else + sessions_q + end + + Enum.reduce(@session_props, sessions_q, fn prop_name, sessions_q -> + prop_val = query.filters["visit:" <> prop_name] + prop_name = if prop_name == "source", do: "referrer_source", else: prop_name + prop_name = if prop_name == "device", do: "screen_size", else: prop_name + prop_name = if prop_name == "os", do: "operating_system", else: prop_name + prop_name = if prop_name == "os_version", do: "operating_system_version", else: prop_name + prop_name = if prop_name == "country", do: "country_code", else: prop_name + + prop_val = + if prop_name == "referrer_source" && prop_val == @no_ref do + "" + else + prop_val + end + + if prop_val do + where_target = [{String.to_existing_atom(prop_name), prop_val}] + from(s in sessions_q, where: ^where_target) + else + sessions_q + end + end) + end + + defp buckets(%Query{interval: "month"} = query) do + n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months) + + Enum.map(n_buckets..0, fn shift -> + query.date_range.last + |> Timex.beginning_of_month() + |> Timex.shift(months: -shift) + end) + end + + defp buckets(%Query{interval: "date"} = query) do + Enum.into(query.date_range, []) + end + + defp select_bucket(q, site, %Query{interval: "month"}) do + from( + e in q, + select: + {fragment("toStartOfMonth(toTimeZone(?, ?)) as bucket", e.timestamp, ^site.timezone), + fragment("uniq(?)", e.user_id)} + ) + end + + defp select_bucket(q, site, %Query{interval: "date"}) do + from( + e in q, + select: + {fragment("toDate(toTimeZone(?, ?)) as bucket", e.timestamp, ^site.timezone), + fragment("uniq(?)", e.user_id)} + ) + end + + defp utc_boundaries(%Query{date_range: date_range}, timezone) do + {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) + + first_datetime = + Timex.to_datetime(first, timezone) + |> Timex.Timezone.convert("UTC") + + {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) + + last_datetime = + Timex.to_datetime(last, timezone) + |> Timex.Timezone.convert("UTC") + + {first_datetime, last_datetime} + end +end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 8f2e9db61..701037472 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,5 +1,5 @@ defmodule Plausible.Stats.Query do - defstruct date_range: nil, step_type: nil, period: nil, steps: nil, filters: %{} + defstruct date_range: nil, interval: nil, period: nil, filters: %{} def shift_back(%__MODULE__{period: "day"} = query) do new_date = query.date_range.first |> Timex.shift(days: -1) @@ -41,30 +41,24 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "realtime", - step_type: "minute", + interval: "minute", date_range: Date.range(date, date), filters: parse_filters(params) } end - def from(_tz, %{"period" => "day", "date" => date} = params) do - date = Date.from_iso8601!(date) - - %__MODULE__{ - period: "day", - date_range: Date.range(date, date), - step_type: "hour", - filters: parse_filters(params) - } - end - def from(tz, %{"period" => "day"} = params) do - date = today(tz) + date = + if params["date"] do + Date.from_iso8601!(params["date"]) + else + today(tz) + end %__MODULE__{ period: "day", date_range: Date.range(date, date), - step_type: "hour", + interval: "hour", filters: parse_filters(params) } end @@ -76,7 +70,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "7d", date_range: Date.range(start_date, end_date), - step_type: "date", + interval: "date", filters: parse_filters(params) } end @@ -88,64 +82,87 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "30d", date_range: Date.range(start_date, end_date), - step_type: "date", + interval: "date", filters: parse_filters(params) } end - def from(_tz, %{"period" => "month", "date" => date} = params) do - start_date = Date.from_iso8601!(date) |> Timex.beginning_of_month() - end_date = Timex.end_of_month(start_date) + def from(tz, %{"period" => "month"} = params) do + date = + if params["date"] do + Date.from_iso8601!(params["date"]) + else + today(tz) + end + + start_date = Timex.beginning_of_month(date) + end_date = Timex.end_of_month(date) %__MODULE__{ period: "month", date_range: Date.range(start_date, end_date), - step_type: "date", - steps: Timex.diff(start_date, end_date, :days), + interval: "date", filters: parse_filters(params) } end def from(tz, %{"period" => "6mo"} = params) do + end_date = + case params["date"] do + nil -> today(tz) + date -> Date.from_iso8601!(date) + end + |> Timex.end_of_month() + start_date = - Timex.shift(today(tz), months: -5) + Timex.shift(end_date, months: -5) |> Timex.beginning_of_month() %__MODULE__{ period: "6mo", - date_range: Date.range(start_date, today(tz)), - step_type: "month", - steps: 6, + date_range: Date.range(start_date, end_date), + interval: Map.get(params, "interval", "month"), filters: parse_filters(params) } end def from(tz, %{"period" => "12mo"} = params) do + end_date = + case params["date"] do + nil -> today(tz) + date -> Date.from_iso8601!(date) + end + |> Timex.end_of_month() + start_date = - Timex.shift(today(tz), months: -11) + Timex.shift(end_date, months: -11) |> Timex.beginning_of_month() %__MODULE__{ period: "12mo", - date_range: Date.range(start_date, today(tz)), - step_type: "month", - steps: 12, + date_range: Date.range(start_date, end_date), + interval: Map.get(params, "interval", "month"), filters: parse_filters(params) } end - def from(_tz, %{"period" => "custom", "from" => from, "to" => to} = params) do + def from(_tz, %{"period" => "custom", "date" => date} = params) do + [from, to] = String.split(date, ",") from_date = Date.from_iso8601!(from) to_date = Date.from_iso8601!(to) %__MODULE__{ period: "custom", date_range: Date.range(from_date, to_date), - step_type: "date", + interval: Map.get(params, "interval", "date"), filters: parse_filters(params) } end + def from(tz, %{"period" => "custom", "from" => from, "to" => to} = params) do + from(tz, Map.merge(params, %{"period" => "custom", "date" => Enum.join([from, to], ",")})) + end + def from(tz, _) do __MODULE__.from(tz, %{"period" => "30d"}) end @@ -154,11 +171,27 @@ defmodule Plausible.Stats.Query do Timex.now(tz) |> Timex.to_date() end - defp parse_filters(params) do - if params["filters"] do - Jason.decode!(params["filters"]) - else - %{} + defp parse_filters(%{"filters" => filters}) when is_binary(filters) do + case Jason.decode(filters) do + {:ok, parsed} -> parsed + {:error, err} -> parse_filter_expression(err.data) end end + + defp parse_filters(%{"filters" => filters}) when is_map(filters), do: filters + defp parse_filters(_), do: %{} + + defp parse_filter_expression(str) do + filters = String.split(str, ";") + + Enum.map(filters, &parse_single_filter/1) + |> Enum.into(%{}) + end + + defp parse_single_filter(str) do + String.trim(str) + |> String.split("==") + |> Enum.map(&String.trim/1) + |> List.to_tuple() + end end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex new file mode 100644 index 000000000..a56ad1a76 --- /dev/null +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -0,0 +1,79 @@ +defmodule PlausibleWeb.Api.ExternalStatsController do + use PlausibleWeb, :controller + use Plausible.Repo + use Plug.ErrorHandler + alias Plausible.Stats.Query + + def realtime_visitors(conn, _params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, %{"period" => "realtime"}) + json(conn, Plausible.Stats.Clickhouse.current_visitors(site, query)) + end + + def aggregate(conn, params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, params) + + metrics = + params["metrics"] + |> String.split(",") + |> Enum.map(&String.trim/1) + + result = + if params["compare"] == "previous_period" do + prev_query = Query.shift_back(query) + + [prev_result, curr_result] = + Task.await_many([ + Task.async(fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end), + Task.async(fn -> Plausible.Stats.aggregate(site, query, metrics) end) + ]) + + Enum.map(curr_result, fn {metric, %{value: current_val}} -> + %{value: prev_val} = prev_result[metric] + + {metric, + %{ + value: current_val, + change: percent_change(prev_val, current_val) + }} + end) + |> Enum.into(%{}) + else + Plausible.Stats.aggregate(site, query, metrics) + end + + json(conn, result) + end + + def timeseries(conn, params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, params) + + {plot, labels} = Plausible.Stats.timeseries(site, query) + + graph = + Enum.zip(labels, plot) + |> Enum.map(fn {label, val} -> %{date: label, value: val} end) + |> Enum.into([]) + + json(conn, graph) + end + + def handle_errors(conn, %{kind: kind, reason: reason}) do + json(conn, %{error: Exception.format_banner(kind, reason)}) + end + + defp percent_change(old_count, new_count) do + cond do + old_count == 0 and new_count > 0 -> + 100 + + old_count == 0 and new_count == 0 -> + 0 + + true -> + round((new_count - old_count) / old_count * 100) + end + end +end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index a94955179..4f7e2a70b 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -3,7 +3,6 @@ defmodule PlausibleWeb.Api.StatsController do use Plausible.Repo alias Plausible.Stats.Clickhouse, as: Stats alias Plausible.Stats.Query - plug PlausibleWeb.AuthorizeStatsPlug def main_graph(conn, params) do site = conn.assigns[:site] @@ -18,7 +17,7 @@ defmodule PlausibleWeb.Api.StatsController do labels: labels, present_index: present_index, top_stats: top_stats, - interval: query.step_type + interval: query.interval }) end diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 144629ded..1262883c4 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -298,6 +298,7 @@ defmodule PlausibleWeb.AuthController do Plausible.Billing.usage_breakdown(conn.assigns[:current_user]) render(conn, "user_settings.html", + user: conn.assigns[:current_user] |> Repo.preload(:api_keys), changeset: changeset, subscription: conn.assigns[:current_user].subscription, theme: conn.assigns[:current_user].theme || "system", @@ -323,6 +324,43 @@ defmodule PlausibleWeb.AuthController do end end + def new_api_key(conn, _params) do + key = :crypto.strong_rand_bytes(64) |> Base.url_encode64() |> binary_part(0, 64) + changeset = Auth.ApiKey.changeset(%Auth.ApiKey{}, %{key: key}) + + render(conn, "new_api_key.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def create_api_key(conn, %{"api_key" => key_params}) do + api_key = %Auth.ApiKey{user_id: conn.assigns[:current_user].id} + changeset = Auth.ApiKey.changeset(api_key, key_params) + + case Repo.insert(changeset) do + {:ok, _api_key} -> + conn + |> put_flash(:success, "API key created successfully") + |> redirect(to: "/settings#api-keys") + + {:error, changeset} -> + render(conn, "new_api_key.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + end + + def delete_api_key(conn, %{"id" => id}) do + Repo.get_by(Auth.ApiKey, id: id) + |> Repo.delete!() + + conn + |> put_flash(:success, "API key revoked successfully") + |> redirect(to: "/settings#api-keys") + end + def delete_me(conn, params) do user = conn.assigns[:current_user] diff --git a/lib/plausible_web/plugs/authorize_api_stats.ex b/lib/plausible_web/plugs/authorize_api_stats.ex new file mode 100644 index 000000000..622215714 --- /dev/null +++ b/lib/plausible_web/plugs/authorize_api_stats.ex @@ -0,0 +1,46 @@ +defmodule PlausibleWeb.AuthorizeApiStatsPlug do + import Plug.Conn + use Plausible.Repo + alias Plausible.Auth.ApiKey + + def init(options) do + options + end + + def call(conn, _opts) do + site = Repo.get_by(Plausible.Site, domain: conn.params["site_id"]) + api_key = get_bearer_token(conn) + + if !(site && api_key) do + not_found(conn) + else + hashed_key = ApiKey.do_hash(api_key) + found_key = Repo.get_by(ApiKey, key_hash: hashed_key) + can_access = found_key && Plausible.Sites.is_owner?(found_key.user_id, site) + + if !can_access do + not_found(conn) + else + assign(conn, :site, site) + end + end + end + + defp get_bearer_token(conn) do + authorization_header = + Plug.Conn.get_req_header(conn, "authorization") + |> List.first() + + case authorization_header do + "Bearer " <> token -> String.trim(token) + _ -> nil + end + end + + defp not_found(conn) do + conn + |> put_status(404) + |> Phoenix.Controller.json(%{error: "Not found"}) + |> halt() + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 13e881c21..60fa223cf 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -26,10 +26,17 @@ defmodule PlausibleWeb.Router do plug PlausibleWeb.AuthPlug end - pipeline :stats_api do + pipeline :internal_stats_api do plug :accepts, ["json"] plug PlausibleWeb.Firewall plug :fetch_session + plug PlausibleWeb.AuthorizeStatsPlug + end + + pipeline :external_stats_api do + plug :accepts, ["json"] + plug PlausibleWeb.Firewall + plug PlausibleWeb.AuthorizeApiStatsPlug end if Application.get_env(:plausible, :environment) == "dev" do @@ -39,7 +46,7 @@ defmodule PlausibleWeb.Router do use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug] scope "/api/stats", PlausibleWeb.Api do - pipe_through :stats_api + pipe_through :internal_stats_api get "/:domain/current-visitors", StatsController, :current_visitors get "/:domain/main-graph", StatsController, :main_graph @@ -61,6 +68,14 @@ defmodule PlausibleWeb.Router do get "/:domain/property/:prop_name", StatsController, :prop_breakdown end + scope "/api/v1/stats", PlausibleWeb.Api do + pipe_through :external_stats_api + + get "/realtime/visitors", ExternalStatsController, :realtime_visitors + get "/aggregate", ExternalStatsController, :aggregate + get "/timeseries", ExternalStatsController, :timeseries + end + scope "/api", PlausibleWeb do pipe_through :api @@ -99,6 +114,9 @@ defmodule PlausibleWeb.Router do get "/settings", AuthController, :user_settings put "/settings", AuthController, :save_settings delete "/me", AuthController, :delete_me + get "/settings/api-keys/new", AuthController, :new_api_key + post "/settings/api-keys", AuthController, :create_api_key + delete "/settings/api-keys/:id", AuthController, :delete_api_key get "/auth/google/callback", AuthController, :google_auth_callback diff --git a/lib/plausible_web/templates/auth/new_api_key.html.eex b/lib/plausible_web/templates/auth/new_api_key.html.eex new file mode 100644 index 000000000..77ada371d --- /dev/null +++ b/lib/plausible_web/templates/auth/new_api_key.html.eex @@ -0,0 +1,21 @@ +<%= form_for @changeset, "/settings/api-keys", [class: "w-full max-w-md mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 py-6 mt-8"], fn f -> %> +

Create new API key

+
+ <%= label f, :name, class: "block text-sm font-medium text-gray-700" %> +
+ <%= text_input f, :name, class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %> +
+ <%= error_tag f, :name %> +
+
+ <%= label f, :key, class: "block text-sm font-medium text-gray-700" %> +
+ <%= text_input f, :key, id: "key-input", class: "shadow-sm bg-gray-50 focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md pr-16", readonly: "readonly" %> + + COPY + +

Make sure to store the key in a secure place. Once created, we will not be able to show it again.

+
+
+ <%= submit "Continue", class: "button mt-4 w-full" %> +<% end %> diff --git a/lib/plausible_web/templates/auth/user_settings.html.eex b/lib/plausible_web/templates/auth/user_settings.html.eex index 335b72bbe..fda898e9f 100644 --- a/lib/plausible_web/templates/auth/user_settings.html.eex +++ b/lib/plausible_web/templates/auth/user_settings.html.eex @@ -1,5 +1,5 @@ <%= if !Application.get_env(:plausible, :is_selfhost) do %> -
+

Subscription Plan

<%= if @subscription do %> @@ -12,10 +12,10 @@
<%= if @subscription && @subscription.status == "deleted" do %> -
-
-
- +
+
+
+

@@ -30,76 +30,76 @@

<% end %> -
-
+
+

Monthly quota

<%= if @subscription do %> -
<%= subscription_quota(@subscription) %> pageviews
+
<%= subscription_quota(@subscription) %> pageviews
<%= case @subscription.status do %> <% "active" -> %> <%= link("Change plan", to: "/billing/change-plan", class: "text-sm text-indigo-500 font-medium") %> <% "past_due" -> %> - Change plan + Change plan <% _ -> %> <% end %> <% else %> -
Free trial
+
Free trial
<%= link("Upgrade", to: "/billing/upgrade", class: "text-sm text-indigo-500 font-medium") %> <% end %>
-
+

Next bill amount

<%= if @subscription && @subscription.status in ["active", "past_due"] do %> -
$<%= @subscription.next_bill_amount %>
+
$<%= @subscription.next_bill_amount %>
<%= if @subscription.update_url do %> <%= link("Update billing info", to: @subscription.update_url, class: "text-sm text-indigo-500 font-medium") %> <% end %> <% else %> -
---
+
---
<% end %>
-
+

Next bill date

<%= if @subscription && @subscription.next_bill_date && @subscription.status in ["active", "past_due"] do %> -
<%= Timex.format!(@subscription.next_bill_date, "{Mshort} {D}, {YYYY}") %>
-
(<%= subscription_interval(@subscription) %> billing)
+
<%= Timex.format!(@subscription.next_bill_date, "{Mshort} {D}, {YYYY}") %>
+
(<%= subscription_interval(@subscription) %> billing)
<% else %> -
---
+
---
<% end %>
-

Your usage

-

Last 30 days total usage across all of your sites

+

Your usage

+

Last 30 days total usage across all of your sites

-
+
- +
- - - - - - @@ -125,7 +125,7 @@ <% end %> -
+

Dashboard Appearance

@@ -139,7 +139,7 @@ <% end %>
-
+

Account settings

@@ -163,7 +163,80 @@ <% end %>
-
+
+

API keys

+ +
+ +
+
+
+ + +
+
+

+ Beta feature +

+
+

+ The stats API is in public beta mode. Feel free to issue API keys and test your integrations. + However, we reserve the right to make breaking changes to the API until we exit the beta and release + v1. +

+
+
+
+
+ +
+
+
+ + <%= if Enum.any?(@user.api_keys) do %> +
+
+ Pageviews + <%= delimit_integer(@usage_pageviews) %>
+ Custom events + <%= delimit_integer(@usage_custom_events) %>
+ Total billable pageviews + <%= delimit_integer(@usage_pageviews + @usage_custom_events) %>
+ + + + + + + + + <%= for api_key <- @user.api_keys do %> + + + + + + <% end %> + +
+ Name + + Key + + Remove +
+ <%= api_key.name %> + + <%= api_key.key_prefix %><%= String.duplicate("*", 32 - 6) %> + + <%= button("Revoke", to: "/settings/api-keys/#{api_key.id}", class: "text-red-600 hover:text-red-900", method: :delete, "data-confirm": "Are you sure you want to revoke this key? This action cannot be reversed.") %> +
+
+ <% end %> + + <%= link "+ New API key", to: "/settings/api-keys/new", class: "button mt-4" %> +
+
+
+
+ +

Delete account

@@ -173,8 +246,8 @@

Deleting your account removes all sites and stats you've collected

<%= if @subscription && @subscription.status == "active" do %> - Delete my account -

Your account cannot be deleted because you have an active subscription. If you want to delete your account, please cancel your subscription first.

+ Delete my account +

Your account cannot be deleted because you have an active subscription. If you want to delete your account, please cancel your subscription first.

<% else %> <%= link("Delete my account", to: "/me", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete", data: [confirm: "Deleting your account cannot be reversed. Are you sure?"]) %> <% end %> diff --git a/priv/repo/migrations/20210128084657_create_api_keys.exs b/priv/repo/migrations/20210128084657_create_api_keys.exs new file mode 100644 index 000000000..029d54d33 --- /dev/null +++ b/priv/repo/migrations/20210128084657_create_api_keys.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.CreateApiKeys do + use Ecto.Migration + + def change do + create table(:api_keys) do + add :user_id, references(:users, on_delete: :delete_all), null: false + add :name, :string, null: false + add :key_prefix, :string, null: false + add :key_hash, :string, null: false + + timestamps() + end + end +end diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index be1f993e5..71830ea8b 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -9,7 +9,7 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-01] - assert q.step_type == "hour" + assert q.interval == "hour" end test "day fromat defaults to today" do @@ -17,7 +17,7 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == Timex.today() assert q.date_range.last == Timex.today() - assert q.step_type == "hour" + assert q.interval == "hour" end test "parses realtime format" do @@ -33,7 +33,7 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-31] - assert q.step_type == "date" + assert q.interval == "date" end test "parses 6 month format" do @@ -42,8 +42,8 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() - assert q.date_range.last == Timex.today() - assert q.step_type == "month" + assert q.date_range.last == Timex.today() |> Timex.end_of_month() + assert q.interval == "month" end test "parses 12 month format" do @@ -52,8 +52,8 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month() - assert q.date_range.last == Timex.today() - assert q.step_type == "month" + assert q.date_range.last == Timex.today() |> Timex.end_of_month() + assert q.interval == "month" end test "defaults to 30 days format" do @@ -65,7 +65,7 @@ defmodule Plausible.Stats.QueryTest do assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-15] - assert q.step_type == "date" + assert q.interval == "date" end describe "filters" do diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs new file mode 100644 index 000000000..f51f6d1b3 --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -0,0 +1,567 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do + use PlausibleWeb.ConnCase + import Plausible.TestUtils + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + @user_id 123 + + test "aggregates a single metric", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 3} + } + end + + test "aggregates visitors, pageviews, bounce rate and visit duration", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 3}, + "visitors" => %{"value" => 2}, + "bounce_rate" => %{"value" => 50}, + "visit_duration" => %{"value" => 750} + } + end + + describe "comparisons" do + test "compare period=day with previous period", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "compare" => "previous_period" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 3, "change" => 200}, + "visitors" => %{"value" => 2, "change" => 100}, + "bounce_rate" => %{"value" => 50, "change" => -50}, + "visit_duration" => %{"value" => 750, "change" => 100} + } + end + end + + describe "filters" do + test "can filter by source", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:source==Google" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by no source/referrer", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Google", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:source==Direct / None" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by referrer", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + referrer: "https://facebook.com", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:referrer==https://facebook.com" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by utm_medium", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_medium: "social", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:utm_medium==social" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by utm_source", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_source: "Twitter", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:utm_source==Twitter" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by utm_campaign", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_campaign: "profile", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:utm_campaign==profile" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by device type", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + screen_size: "Desktop", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:device==Desktop" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by browser", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + browser: "Chrome", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:browser==Chrome" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by browser version", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + browser: "Chrome", + browser_version: "56", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:browser_version==56" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by operating system", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + operating_system: "Mac", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:os==Mac" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by operating system version", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + operating_system_version: "10.5", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:os_version==10.5" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "can filter by country", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + country_code: "EE", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "visit:country==EE" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + + test "when filtering by page, session metrics treat is like entry_page", %{ + conn: conn, + site: site + } do + populate_stats([ + build(:pageview, + pathname: "/blogpost", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "event:page==/blogpost" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 2}, + "visitors" => %{"value" => 2}, + "bounce_rate" => %{"value" => 50}, + "visit_duration" => %{"value" => 750} + } + end + + test "combining filters", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + pathname: "/blogpost", + country_code: "EE", + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + domain: site.domain, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "metrics" => "pageviews,visitors,bounce_rate,visit_duration", + "filters" => "event:page==/blogpost;visit:country==EE" + }) + + assert json_response(conn, 200) == %{ + "pageviews" => %{"value" => 1}, + "visitors" => %{"value" => 1}, + "bounce_rate" => %{"value" => 0}, + "visit_duration" => %{"value" => 1500} + } + end + end +end diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs new file mode 100644 index 000000000..c9b66f08a --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs @@ -0,0 +1,397 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do + use PlausibleWeb.ConnCase + import Plausible.TestUtils + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + test "shows last 6 months of visitors", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "6mo", + "date" => "2021-01-01" + }) + + assert json_response(conn, 200) == [ + %{"date" => "2020-08-01", "value" => 0}, + %{"date" => "2020-09-01", "value" => 0}, + %{"date" => "2020-10-01", "value" => 0}, + %{"date" => "2020-11-01", "value" => 0}, + %{"date" => "2020-12-01", "value" => 1}, + %{"date" => "2021-01-01", "value" => 2} + ] + end + + test "shows last 12 months of visitors", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "12mo", + "date" => "2021-01-01" + }) + + assert json_response(conn, 200) == [ + %{"date" => "2020-02-01", "value" => 1}, + %{"date" => "2020-03-01", "value" => 0}, + %{"date" => "2020-04-01", "value" => 0}, + %{"date" => "2020-05-01", "value" => 0}, + %{"date" => "2020-06-01", "value" => 0}, + %{"date" => "2020-07-01", "value" => 0}, + %{"date" => "2020-08-01", "value" => 0}, + %{"date" => "2020-09-01", "value" => 0}, + %{"date" => "2020-10-01", "value" => 0}, + %{"date" => "2020-11-01", "value" => 0}, + %{"date" => "2020-12-01", "value" => 1}, + %{"date" => "2021-01-01", "value" => 2} + ] + end + + test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "12mo", + "interval" => "date" + }) + + res = json_response(conn, 200) + assert Enum.count(res) in [365, 366] + end + + test "shows a custom range with daily interval", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "custom", + "from" => "2021-01-01", + "to" => "2021-01-02" + }) + + assert json_response(conn, 200) == [ + %{"date" => "2021-01-01", "value" => 2}, + %{"date" => "2021-01-02", "value" => 1} + ] + end + + test "shows a custom range with monthly interval", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2020-12-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "custom", + "from" => "2020-12-01", + "to" => "2021-01-02", + "interval" => "month" + }) + + assert json_response(conn, 200) == [ + %{"date" => "2020-12-01", "value" => 1}, + %{"date" => "2021-01-01", "value" => 2} + ] + end + + describe "filters" do + test "can filter by source", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + referrer_source: "Google", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:source==Google" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by no source/referrer", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + referrer_source: "Google", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:source==Direct / None" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by referrer", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + referrer: "https://facebook.com", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:referrer==https://facebook.com" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by utm_medium", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_medium: "social", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:utm_medium==social" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by utm_source", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_source: "Twitter", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:utm_source==Twitter" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by utm_campaign", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + utm_campaign: "profile", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:utm_campaign==profile" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by device type", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + screen_size: "Desktop", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:device==Desktop" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by browser", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + browser: "Chrome", + browser_version: "56.1", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + browser_version: "55", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:browser==Chrome;visit:browser_version==56.1" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by operating system", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + operating_system: "Mac", + operating_system_version: "10.5", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + operating_system: "Something else", + operating_system_version: "10.5", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + operating_system: "Mac", + operating_system_version: "10.4", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:os == Mac;visit:os_version==10.5" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by country", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + country_code: "EE", + operating_system_version: "10.5", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "visit:country==EE" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 1} + end + + test "can filter by page", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + pathname: "/hello", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/hello", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/goobye", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "period" => "month", + "date" => "2021-01-01", + "filters" => "event:page==/hello" + }) + + res = json_response(conn, 200) + assert List.first(res) == %{"date" => "2021-01-01", "value" => 2} + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 3b3598193..78e36018d 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -151,6 +151,17 @@ defmodule Plausible.Factory do } end + def api_key_factory do + key = :crypto.strong_rand_bytes(64) |> Base.url_encode64() |> binary_part(0, 64) + + %Plausible.Auth.ApiKey{ + name: "api-key-name", + key: key, + key_hash: Plausible.Auth.ApiKey.do_hash(key), + key_prefix: binary_part(key, 0, 6) + } + end + defp hash_key() do Keyword.fetch!( Application.get_env(:plausible, PlausibleWeb.Endpoint), diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 6f7acbb49..6b4c7706d 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -11,6 +11,23 @@ defmodule Plausible.TestUtils do {:ok, site: site} end + def create_new_site(%{user: user}) do + site = Factory.insert(:site, members: [user]) + {:ok, site: site} + end + + def create_api_key(%{user: user}) do + api_key = Factory.insert(:api_key, user: user) + + {:ok, api_key: api_key.key} + end + + def use_api_key(%{conn: conn, api_key: api_key}) do + conn = Plug.Conn.put_req_header(conn, "authorization", "Bearer #{api_key}") + + {:ok, conn: conn} + end + def create_pageviews(pageviews) do pageviews = Enum.map(pageviews, fn pageview -> @@ -61,4 +78,31 @@ defmodule Plausible.TestUtils do |> Plug.Session.call(opts) |> Plug.Conn.fetch_session() end + + def populate_stats(events) do + sessions = + Enum.reduce(events, %{}, fn event, sessions -> + Plausible.Session.Store.reconcile_event(sessions, event) + end) + + events = + Enum.map(events, fn event -> + Map.put(event, :session_id, sessions[event.user_id].session_id) + end) + + Plausible.ClickhouseRepo.insert_all( + Plausible.ClickhouseEvent, + Enum.map(events, &schema_to_map/1) + ) + + Plausible.ClickhouseRepo.insert_all( + Plausible.ClickhouseSession, + Enum.map(Map.values(sessions), &schema_to_map/1) + ) + end + + defp schema_to_map(schema) do + Map.from_struct(schema) + |> Map.delete(:__meta__) + end end