mirror of
https://github.com/plausible/analytics.git
synced 2025-01-09 03:26:52 +03:00
Remove stats from postgres (#74)
* Remove stats code related to postgres events and sessions * Use Clickhouse events in fetch tweets worker
This commit is contained in:
Uku Taht
GitHub
parent
390a4577b5
commit
933cff6fe0
@ -2,6 +2,7 @@ defmodule Plausible.ClickhouseEvent do
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key false
|
||||
schema "events" do
|
||||
field :name, :string
|
||||
field :domain, :string
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
defmodule Plausible.Event do
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
schema "events" do
|
||||
field :name, :string
|
||||
field :domain, :string
|
||||
field :hostname, :string
|
||||
field :pathname, :string
|
||||
field :fingerprint, :string
|
||||
|
||||
field :referrer, :string
|
||||
field :referrer_source, :string
|
||||
field :initial_referrer, :string
|
||||
field :initial_referrer_source, :string
|
||||
field :country_code, :string
|
||||
field :screen_size, :string
|
||||
field :operating_system, :string
|
||||
field :browser, :string
|
||||
|
||||
timestamps(inserted_at: :timestamp, updated_at: false)
|
||||
end
|
||||
|
||||
def changeset(pageview, attrs) do
|
||||
pageview
|
||||
|> cast(attrs, [:name, :domain, :hostname, :pathname, :fingerprint, :operating_system, :browser, :referrer, :referrer_source, :initial_referrer, :initial_referrer_source, :country_code, :screen_size])
|
||||
|> validate_required([:name, :domain, :hostname, :pathname, :fingerprint])
|
||||
end
|
||||
end
|
||||
@ -20,13 +20,18 @@ defmodule Plausible.Event.WriteBuffer do
|
||||
{:ok, event}
|
||||
end
|
||||
|
||||
def flush() do
|
||||
GenServer.call(__MODULE__, :flush, :infinity)
|
||||
:ok
|
||||
end
|
||||
|
||||
def handle_cast({:insert, event}, %{buffer: buffer} = state) do
|
||||
new_buffer = [ event | buffer ]
|
||||
|
||||
if length(new_buffer) >= @max_buffer_size do
|
||||
Logger.info("Buffer full, flushing to disk")
|
||||
Process.cancel_timer(state[:timer])
|
||||
flush(new_buffer)
|
||||
do_flush(new_buffer)
|
||||
new_timer = Process.send_after(self(), :tick, @flush_interval_ms)
|
||||
{:noreply, %{buffer: [], timer: new_timer}}
|
||||
else
|
||||
@ -35,17 +40,24 @@ defmodule Plausible.Event.WriteBuffer do
|
||||
end
|
||||
|
||||
def handle_info(:tick, %{buffer: buffer}) do
|
||||
flush(buffer)
|
||||
do_flush(buffer)
|
||||
timer = Process.send_after(self(), :tick, @flush_interval_ms)
|
||||
{:noreply, %{buffer: [], timer: timer}}
|
||||
end
|
||||
|
||||
def terminate(_reason, %{buffer: buffer}) do
|
||||
Logger.info("Flushing event buffer before shutdown...")
|
||||
flush(buffer)
|
||||
def handle_call(:flush, _from, %{buffer: buffer} = state) do
|
||||
Process.cancel_timer(state[:timer])
|
||||
do_flush(buffer)
|
||||
new_timer = Process.send_after(self(), :tick, @flush_interval_ms)
|
||||
{:reply, nil, %{buffer: [], timer: new_timer}}
|
||||
end
|
||||
|
||||
defp flush(buffer) do
|
||||
def terminate(_reason, %{buffer: buffer}) do
|
||||
Logger.info("Flushing event buffer before shutdown...")
|
||||
do_flush(buffer)
|
||||
end
|
||||
|
||||
defp do_flush(buffer) do
|
||||
case buffer do
|
||||
[] -> nil
|
||||
events ->
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
defmodule Plausible.Ingest.FingerprintSession do
|
||||
use GenServer
|
||||
use Plausible.Repo
|
||||
|
||||
@session_timeout Application.get_env(:plausible, :session_timeout)
|
||||
|
||||
def on_event(event) do
|
||||
user_session = :global.whereis_name(event.fingerprint)
|
||||
|
||||
if is_pid(user_session) do
|
||||
GenServer.cast(user_session, {:on_event, event})
|
||||
else
|
||||
GenServer.start_link(__MODULE__, event, name: {:global, event.fingerprint})
|
||||
end
|
||||
end
|
||||
|
||||
def on_unload(fingerprint, timestamp) do
|
||||
user_session = :global.whereis_name(fingerprint)
|
||||
|
||||
if is_pid(user_session) do
|
||||
GenServer.cast(user_session, {:on_unload, timestamp})
|
||||
end
|
||||
end
|
||||
|
||||
def init(event) do
|
||||
timer = Process.send_after(self(), :finalize, @session_timeout)
|
||||
{:ok, %{first_event: event, last_event: event, timer: timer, is_bounce: true, last_unload: nil}}
|
||||
end
|
||||
|
||||
def handle_cast({:on_event, event}, state) do
|
||||
Process.cancel_timer(state[:timer])
|
||||
new_timer = Process.send_after(self(), :finalize, @session_timeout)
|
||||
{:noreply, %{state | timer: new_timer, last_event: event, is_bounce: false, last_unload: nil}}
|
||||
end
|
||||
|
||||
def handle_cast({:on_unload, timestamp}, state) do
|
||||
{:noreply, %{state | last_unload: timestamp}}
|
||||
end
|
||||
|
||||
def handle_info(:finalize, state) do
|
||||
first_event = state[:first_event]
|
||||
last_event = state[:last_event]
|
||||
|
||||
if !is_potential_leftover?(first_event) do
|
||||
length = if state[:last_unload] do
|
||||
Timex.diff(state[:last_unload], first_event.timestamp, :seconds)
|
||||
end
|
||||
|
||||
changeset = Plausible.FingerprintSession.changeset(%Plausible.FingerprintSession{}, %{
|
||||
hostname: first_event.hostname,
|
||||
domain: first_event.domain,
|
||||
fingerprint: first_event.fingerprint,
|
||||
entry_page: first_event.pathname,
|
||||
exit_page: last_event.pathname,
|
||||
is_bounce: state[:is_bounce],
|
||||
length: length,
|
||||
referrer: first_event.referrer,
|
||||
referrer_source: first_event.referrer_source,
|
||||
country_code: first_event.country_code,
|
||||
operating_system: first_event.operating_system,
|
||||
browser: first_event.browser,
|
||||
start: first_event.timestamp
|
||||
})
|
||||
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
defp is_potential_leftover?(%{timestamp: timestamp}) do
|
||||
server_start_time = Application.get_env(:plausible, :server_start)
|
||||
Timex.diff(timestamp, server_start_time, :milliseconds) < @session_timeout
|
||||
end
|
||||
|
||||
end
|
||||
@ -1,31 +0,0 @@
|
||||
defmodule Plausible.FingerprintSession do
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
schema "fingerprint_sessions" do
|
||||
field :hostname, :string
|
||||
field :domain, :string
|
||||
field :fingerprint, :string
|
||||
|
||||
field :start, :naive_datetime, null: false
|
||||
field :length, :integer
|
||||
field :is_bounce, :boolean
|
||||
field :entry_page, :string
|
||||
field :exit_page, :string
|
||||
|
||||
field :referrer, :string
|
||||
field :referrer_source, :string
|
||||
field :country_code, :string
|
||||
field :screen_size, :string
|
||||
field :operating_system, :string
|
||||
field :browser, :string
|
||||
|
||||
timestamps(inserted_at: :timestamp, updated_at: false)
|
||||
end
|
||||
|
||||
def changeset(session, attrs) do
|
||||
session
|
||||
|> cast(attrs, [:hostname, :domain, :entry_page, :exit_page, :referrer, :fingerprint, :start, :length, :is_bounce, :operating_system, :browser, :referrer_source, :country_code, :screen_size])
|
||||
|> validate_required([:hostname, :domain, :fingerprint, :is_bounce, :start])
|
||||
end
|
||||
end
|
||||
@ -1,555 +0,0 @@
|
||||
defmodule Plausible.Stats do
|
||||
use Plausible.Repo
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def compare_pageviews_and_visitors(site, query, {pageviews, visitors}) do
|
||||
query = Query.shift_back(query)
|
||||
{old_pageviews, old_visitors} = pageviews_and_visitors(site, query)
|
||||
|
||||
cond do
|
||||
old_pageviews == 0 and pageviews > 0 ->
|
||||
{100, 100}
|
||||
old_pageviews == 0 and pageviews == 0 ->
|
||||
{0, 0}
|
||||
true ->
|
||||
{
|
||||
round((pageviews - old_pageviews) / old_pageviews * 100),
|
||||
round((visitors - old_visitors) / old_visitors * 100),
|
||||
}
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def calculate_plot(site, %Query{step_type: "month"} = query) do
|
||||
steps = Enum.map((query.steps - 1)..0, fn shift ->
|
||||
Timex.now(site.timezone)
|
||||
|> Timex.beginning_of_month
|
||||
|> Timex.shift(months: -shift)
|
||||
|> DateTime.to_date
|
||||
end)
|
||||
|
||||
groups = Repo.all(
|
||||
from e in base_query(site, %{query | filters: %{}}),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('month', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
) |> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.to_date(dt) end)
|
||||
|
||||
compare_groups = if query.filters["goal"] do
|
||||
Repo.all(
|
||||
from e in base_query(site, query),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('month', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
) |> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.to_date(dt) end)
|
||||
end
|
||||
|
||||
present_index = Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date |> Timex.beginning_of_month end)
|
||||
plot = Enum.map(steps, fn step -> groups[step] || 0 end)
|
||||
compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end)
|
||||
labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end)
|
||||
|
||||
{plot, compare_plot, labels, present_index}
|
||||
end
|
||||
|
||||
def calculate_plot(site, %Query{step_type: "date"} = query) do
|
||||
steps = Enum.into(query.date_range, [])
|
||||
|
||||
groups = Repo.all(
|
||||
from e in base_query(site, %{ query | filters: %{} }),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('day', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
) |> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.to_date(dt) end)
|
||||
|
||||
compare_groups = if query.filters["goal"] do
|
||||
Repo.all(
|
||||
from e in base_query(site, query),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('day', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
) |> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.to_date(dt) end)
|
||||
end
|
||||
|
||||
present_index = Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date end)
|
||||
steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps)
|
||||
plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show)
|
||||
compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end)
|
||||
labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end)
|
||||
|
||||
{plot, compare_plot, labels, present_index}
|
||||
end
|
||||
|
||||
def calculate_plot(site, %Query{step_type: "hour"} = query) do
|
||||
{:ok, beginning_of_day} = NaiveDateTime.new(query.date_range.first, ~T[00:00:00])
|
||||
|
||||
steps = Enum.map(0..23, fn shift ->
|
||||
beginning_of_day
|
||||
|> Timex.shift(hours: shift)
|
||||
|> truncate_to_hour
|
||||
|> NaiveDateTime.truncate(:second)
|
||||
end)
|
||||
|
||||
groups = Repo.all(
|
||||
from e in base_query(site, %{query | filters: %{}}),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('hour', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
)
|
||||
|> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.truncate(dt, :second) end)
|
||||
|
||||
compare_groups = if query.filters["goal"] do
|
||||
Repo.all(
|
||||
from e in base_query(site, query),
|
||||
group_by: 1,
|
||||
order_by: 1,
|
||||
select: {fragment("date_trunc('hour', ? at time zone 'utc' at time zone ?)", e.timestamp, ^site.timezone), count(e.fingerprint, :distinct)}
|
||||
)
|
||||
|> Enum.into(%{})
|
||||
|> transform_keys(fn dt -> NaiveDateTime.truncate(dt, :second) end)
|
||||
end
|
||||
|
||||
present_index = Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> truncate_to_hour |> NaiveDateTime.truncate(:second) end)
|
||||
steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps)
|
||||
plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show)
|
||||
compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end)
|
||||
labels = Enum.map(steps, fn step -> NaiveDateTime.to_iso8601(step) end)
|
||||
{plot, compare_plot, labels, present_index}
|
||||
end
|
||||
|
||||
def bounce_rate(site, query) do
|
||||
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
|
||||
|
||||
sessions_query = from(s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime
|
||||
)
|
||||
total_sessions = Repo.one( from s in sessions_query, select: count(s))
|
||||
bounced_sessions = Repo.one(from s in sessions_query, where: s.is_bounce, select: count(s))
|
||||
|
||||
case total_sessions do
|
||||
0 -> 0
|
||||
total -> round(bounced_sessions / total * 100)
|
||||
end
|
||||
end
|
||||
|
||||
def pageviews_and_visitors(site, query) do
|
||||
Repo.one(from(
|
||||
e in base_query(site, query),
|
||||
select: {count(e.id), count(e.fingerprint, :distinct)}
|
||||
))
|
||||
end
|
||||
|
||||
def unique_visitors(site, query) do
|
||||
Repo.one(from(
|
||||
e in base_query(site, query),
|
||||
select: count(e.fingerprint, :distinct)
|
||||
))
|
||||
end
|
||||
|
||||
def top_referrers_for_goal(site, query, limit \\ 5) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.initial_referrer_source, url: min(e.initial_referrer), count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.initial_referrer_source,
|
||||
where: not is_nil(e.initial_referrer_source),
|
||||
order_by: [desc: 3],
|
||||
limit: ^limit
|
||||
) |> Enum.map(fn ref ->
|
||||
Map.update(ref, :url, nil, fn url -> url && URI.parse("http://" <> url).host end)
|
||||
end)
|
||||
end
|
||||
|
||||
def top_referrers(site, query, limit \\ 5, include \\ []) do
|
||||
referrers = Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.referrer_source, url: min(e.referrer), count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.referrer_source,
|
||||
where: not is_nil(e.referrer_source),
|
||||
order_by: [desc: 3],
|
||||
limit: ^limit
|
||||
) |> Enum.map(fn ref ->
|
||||
Map.update(ref, :url, nil, fn url -> url && URI.parse("http://" <> url).host end)
|
||||
end)
|
||||
|
||||
if "bounce_rate" in include do
|
||||
bounce_rates = bounce_rates_by_referrer_source(site, query, Enum.map(referrers, fn ref -> ref[:name] end))
|
||||
|
||||
Enum.map(referrers, fn referrer ->
|
||||
Map.put(referrer, :bounce_rate, bounce_rates[referrer[:name]])
|
||||
end)
|
||||
else
|
||||
referrers
|
||||
end
|
||||
end
|
||||
|
||||
defp bounce_rates_by_referrer_source(site, query, referrers) do
|
||||
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
|
||||
|
||||
total_sessions_by_referrer = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.referrer_source in ^referrers,
|
||||
group_by: s.referrer_source,
|
||||
select: {s.referrer_source, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
bounced_sessions_by_referrer = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.is_bounce,
|
||||
where: s.referrer_source in ^referrers,
|
||||
group_by: s.referrer_source,
|
||||
select: {s.referrer_source, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
Enum.reduce(referrers, %{}, fn referrer, acc ->
|
||||
total_sessions = Map.get(total_sessions_by_referrer, referrer, 0)
|
||||
bounced_sessions = Map.get(bounced_sessions_by_referrer, referrer, 0)
|
||||
|
||||
bounce_rate = if total_sessions > 0 do
|
||||
round(bounced_sessions / total_sessions * 100)
|
||||
end
|
||||
|
||||
Map.put(acc, referrer, bounce_rate)
|
||||
end)
|
||||
end
|
||||
|
||||
def visitors_from_referrer(site, query, referrer) do
|
||||
Repo.one(
|
||||
from e in base_query(site, query),
|
||||
select: count(e.fingerprint, :distinct),
|
||||
where: e.referrer_source == ^referrer
|
||||
)
|
||||
end
|
||||
|
||||
def conversions_from_referrer(site, query, referrer) do
|
||||
Repo.one(
|
||||
from e in base_query(site, query),
|
||||
select: count(e.fingerprint, :distinct),
|
||||
where: e.initial_referrer_source == ^referrer
|
||||
)
|
||||
end
|
||||
|
||||
def referrer_drilldown(site, query, referrer, include \\ []) do
|
||||
referring_urls = Repo.all(
|
||||
from e in base_query(site, query),
|
||||
select: %{name: e.referrer, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.referrer,
|
||||
where: e.referrer_source == ^referrer,
|
||||
order_by: [desc: 2],
|
||||
limit: 100
|
||||
)
|
||||
|
||||
referring_urls = if "bounce_rate" in include do
|
||||
bounce_rates = bounce_rates_by_referring_url(site, query, Enum.map(referring_urls, fn ref -> ref[:name] end))
|
||||
|
||||
Enum.map(referring_urls, fn url ->
|
||||
Map.put(url, :bounce_rate, bounce_rates[url[:name]])
|
||||
end)
|
||||
else
|
||||
referring_urls
|
||||
end
|
||||
|
||||
if referrer == "Twitter" do
|
||||
urls = Enum.map(referring_urls, &(&1[:name]))
|
||||
|
||||
tweets = Repo.all(
|
||||
from t in Plausible.Twitter.Tweet,
|
||||
where: t.link in ^urls
|
||||
) |> Enum.group_by(&(&1.link))
|
||||
|
||||
Enum.map(referring_urls, fn url ->
|
||||
Map.put(url, :tweets, tweets[url[:name]])
|
||||
end)
|
||||
else
|
||||
referring_urls
|
||||
end
|
||||
end
|
||||
|
||||
def referrer_drilldown_for_goal(site, query, referrer) do
|
||||
Repo.all(
|
||||
from e in base_query(site, query),
|
||||
select: %{name: e.initial_referrer, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.initial_referrer,
|
||||
where: e.initial_referrer_source == ^referrer,
|
||||
order_by: [desc: 2],
|
||||
limit: 100
|
||||
)
|
||||
end
|
||||
|
||||
defp bounce_rates_by_referring_url(site, query, referring_urls) do
|
||||
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
|
||||
|
||||
total_sessions_by_url = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.referrer in ^referring_urls,
|
||||
group_by: s.referrer,
|
||||
select: {s.referrer, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
bounced_sessions_by_url = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.is_bounce,
|
||||
where: s.referrer in ^referring_urls,
|
||||
group_by: s.referrer,
|
||||
select: {s.referrer, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
Enum.reduce(referring_urls, %{}, fn url, acc ->
|
||||
total_sessions = Map.get(total_sessions_by_url, url, 0)
|
||||
bounced_sessions = Map.get(bounced_sessions_by_url, url, 0)
|
||||
|
||||
bounce_rate = if total_sessions > 0 do
|
||||
round(bounced_sessions / total_sessions * 100)
|
||||
end
|
||||
|
||||
Map.put(acc, url, bounce_rate)
|
||||
end)
|
||||
end
|
||||
|
||||
def top_pages(site, query, limit \\ 5, include \\ []) do
|
||||
pages = Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.pathname, count: count(e.pathname)},
|
||||
group_by: e.pathname,
|
||||
order_by: [desc: count(e.pathname)],
|
||||
limit: ^limit
|
||||
)
|
||||
|
||||
if "bounce_rate" in include do
|
||||
bounce_rates = bounce_rates_by_page_url(site, query, Enum.map(pages, fn page -> page[:name] end))
|
||||
|
||||
Enum.map(pages, fn url ->
|
||||
Map.put(url, :bounce_rate, bounce_rates[url[:name]])
|
||||
end)
|
||||
else
|
||||
pages
|
||||
end
|
||||
end
|
||||
|
||||
defp bounce_rates_by_page_url(site, query, page_urls) do
|
||||
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
|
||||
|
||||
total_sessions_by_url = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.entry_page in ^page_urls,
|
||||
group_by: s.entry_page,
|
||||
select: {s.entry_page, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
bounced_sessions_by_url = Repo.all(
|
||||
from s in Plausible.FingerprintSession,
|
||||
where: s.domain == ^site.domain,
|
||||
where: s.start >= ^first_datetime and s.start < ^last_datetime,
|
||||
where: s.is_bounce,
|
||||
where: s.entry_page in ^page_urls,
|
||||
group_by: s.entry_page,
|
||||
select: {s.entry_page, count(s.id)}
|
||||
) |> Enum.into(%{})
|
||||
|
||||
Enum.reduce(page_urls, %{}, fn url, acc ->
|
||||
total_sessions = Map.get(total_sessions_by_url, url, 0)
|
||||
bounced_sessions = Map.get(bounced_sessions_by_url, url, 0)
|
||||
|
||||
bounce_rate = if total_sessions > 0 do
|
||||
round(bounced_sessions / total_sessions * 100)
|
||||
end
|
||||
|
||||
Map.put(acc, url, bounce_rate)
|
||||
end)
|
||||
end
|
||||
|
||||
defp add_percentages(stat_list) do
|
||||
total = Enum.reduce(stat_list, 0, fn %{count: count}, total -> total + count end)
|
||||
Enum.map(stat_list, fn stat ->
|
||||
Map.put(stat, :percentage, round(stat[:count] / total * 100))
|
||||
end)
|
||||
end
|
||||
|
||||
@available_screen_sizes ["Desktop", "Laptop", "Tablet", "Mobile"]
|
||||
|
||||
def top_screen_sizes(site, query) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.screen_size, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.screen_size,
|
||||
where: not is_nil(e.screen_size)
|
||||
)
|
||||
|> Enum.sort(fn %{name: screen_size1}, %{name: screen_size2} ->
|
||||
index1 = Enum.find_index(@available_screen_sizes, fn s -> s == screen_size1 end)
|
||||
index2 = Enum.find_index(@available_screen_sizes, fn s -> s == screen_size2 end)
|
||||
index2 > index1
|
||||
end)
|
||||
|> add_percentages
|
||||
end
|
||||
|
||||
def countries(site, query) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.country_code, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.country_code,
|
||||
where: not is_nil(e.country_code),
|
||||
order_by: [desc: 2]
|
||||
)
|
||||
|> Enum.map(fn stat ->
|
||||
two_letter_code = stat[:name]
|
||||
stat
|
||||
|> Map.put(:name, Plausible.Stats.CountryName.to_alpha3(two_letter_code))
|
||||
|> Map.put(:full_country_name, Plausible.Stats.CountryName.from_iso3166(two_letter_code))
|
||||
end)
|
||||
|> add_percentages
|
||||
end
|
||||
|
||||
def browsers(site, query, limit \\ 5) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.browser, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.browser,
|
||||
where: not is_nil(e.browser),
|
||||
order_by: [desc: 2]
|
||||
)
|
||||
|> add_percentages
|
||||
|> Enum.take(limit)
|
||||
end
|
||||
|
||||
def operating_systems(site, query, limit \\ 5) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: %{name: e.operating_system, count: count(e.fingerprint, :distinct)},
|
||||
group_by: e.operating_system,
|
||||
where: not is_nil(e.operating_system),
|
||||
order_by: [desc: 2]
|
||||
)
|
||||
|> add_percentages
|
||||
|> Enum.take(limit)
|
||||
end
|
||||
|
||||
def current_visitors(site) do
|
||||
Repo.one(
|
||||
from e in Plausible.Event,
|
||||
where: e.timestamp >= fragment("(now() at time zone 'utc') - '5 minutes'::interval"),
|
||||
where: e.domain == ^site.domain,
|
||||
select: count(e.fingerprint, :distinct)
|
||||
)
|
||||
end
|
||||
|
||||
def has_pageviews?(site) do
|
||||
Repo.exists?(
|
||||
from e in Plausible.Event,
|
||||
where: e.domain == ^site.domain
|
||||
)
|
||||
end
|
||||
|
||||
def goal_conversions(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do
|
||||
Repo.all(from e in base_query(site, query),
|
||||
select: count(e.fingerprint, :distinct),
|
||||
group_by: e.name,
|
||||
order_by: [desc: 1]
|
||||
) |> Enum.map(fn count -> %{name: goal, count: count} end)
|
||||
end
|
||||
|
||||
def goal_conversions(site, query) do
|
||||
goals = Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain)
|
||||
fetch_pageview_goals(goals, site, query)
|
||||
++ fetch_event_goals(goals, site, query)
|
||||
|> sort_conversions()
|
||||
end
|
||||
|
||||
defp fetch_event_goals(goals, site, query) do
|
||||
events = Enum.map(goals, fn goal -> goal.event_name end)
|
||||
|> Enum.filter(&(&1))
|
||||
|
||||
if Enum.count(events) > 0 do
|
||||
Repo.all(
|
||||
from e in base_query(site, query, events),
|
||||
group_by: e.name,
|
||||
select: %{name: e.name, count: count(e.fingerprint, :distinct)}
|
||||
)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp fetch_pageview_goals(goals, site, query) do
|
||||
pages = Enum.map(goals, fn goal -> goal.page_path end)
|
||||
|> Enum.filter(&(&1))
|
||||
|
||||
if Enum.count(pages) > 0 do
|
||||
Repo.all(
|
||||
from e in base_query(site, query),
|
||||
where: e.pathname in ^pages,
|
||||
group_by: e.pathname,
|
||||
select: %{name: fragment("concat('Visit ', ?)", e.pathname), count: count(e.fingerprint, :distinct)}
|
||||
)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp sort_conversions(conversions) do
|
||||
Enum.sort_by(conversions, fn conversion -> -conversion[:count] end)
|
||||
end
|
||||
|
||||
defp base_query(site, query, events \\ ["pageview"]) do
|
||||
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
|
||||
{goal_event, path} = event_name_for_goal(query)
|
||||
|
||||
q = from(e in Plausible.Event,
|
||||
where: e.domain == ^site.domain,
|
||||
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
|
||||
)
|
||||
|
||||
q = if path do
|
||||
from(e in q, where: e.pathname == ^path)
|
||||
else
|
||||
q
|
||||
end
|
||||
|
||||
if goal_event do
|
||||
from(e in q, where: e.name == ^goal_event)
|
||||
else
|
||||
from(e in q, where: e.name in ^events)
|
||||
end
|
||||
end
|
||||
|
||||
defp date_range_utc_boundaries(date_range, timezone) do
|
||||
{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
|
||||
first_datetime = Timex.to_datetime(first, timezone)
|
||||
|> Timex.Timezone.convert("UTC")
|
||||
|
||||
{:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])
|
||||
last_datetime = Timex.to_datetime(last, timezone)
|
||||
|> Timex.Timezone.convert("UTC")
|
||||
|
||||
{first_datetime, last_datetime}
|
||||
end
|
||||
|
||||
defp event_name_for_goal(query) do
|
||||
case query.filters["goal"] do
|
||||
"Visit " <> page ->
|
||||
{"pageview", page}
|
||||
goal when is_binary(goal) ->
|
||||
{goal, nil}
|
||||
_ ->
|
||||
{nil, nil}
|
||||
end
|
||||
end
|
||||
|
||||
defp transform_keys(map, fun) do
|
||||
for {key, val} <- map, into: %{} do
|
||||
{fun.(key), val}
|
||||
end
|
||||
end
|
||||
|
||||
defp truncate_to_hour(datetime) do
|
||||
{:ok, datetime} = NaiveDateTime.new(datetime.year, datetime.month, datetime.day, datetime.hour, 0, 0, 0)
|
||||
datetime
|
||||
end
|
||||
end
|
||||
15
lib/plausible/twitter/api.ex
Normal file
15
lib/plausible/twitter/api.ex
Normal file
@ -0,0 +1,15 @@
|
||||
defmodule Plausible.Twitter.Api do
|
||||
def search(link) do
|
||||
params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
|
||||
params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, oauth_credentials())
|
||||
uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
|
||||
response = HTTPoison.get!(uri)
|
||||
Jason.decode!(response.body)
|
||||
|> Map.get("statuses")
|
||||
end
|
||||
|
||||
defp oauth_credentials() do
|
||||
Application.get_env(:plausible, :twitter, %{})
|
||||
|> OAuther.credentials()
|
||||
end
|
||||
end
|
||||
@ -6,10 +6,7 @@ defmodule PlausibleWeb.Api.ExternalController do
|
||||
params = parse_body(conn)
|
||||
|
||||
case create_event(conn, params) do
|
||||
{:ok, nil} ->
|
||||
conn |> send_resp(202, "")
|
||||
{:ok, event} ->
|
||||
Plausible.Ingest.FingerprintSession.on_event(event)
|
||||
{:ok, _} ->
|
||||
conn |> send_resp(202, "")
|
||||
{:error, changeset} ->
|
||||
request = Sentry.Plug.build_request_interface_data(conn, [])
|
||||
@ -19,13 +16,6 @@ defmodule PlausibleWeb.Api.ExternalController do
|
||||
end
|
||||
end
|
||||
|
||||
def unload(conn, _params) do
|
||||
params = parse_body(conn)
|
||||
fingerprint = calculate_fingerprint(conn, params)
|
||||
Plausible.Ingest.FingerprintSession.on_unload(fingerprint, Timex.now())
|
||||
conn |> send_resp(202, "")
|
||||
end
|
||||
|
||||
def error(conn, _params) do
|
||||
request = Sentry.Plug.build_request_interface_data(conn, [])
|
||||
Sentry.capture_message("JS snippet error", request: request)
|
||||
@ -47,12 +37,13 @@ defmodule PlausibleWeb.Api.ExternalController do
|
||||
initial_ref = parse_referrer(uri, params["initial_referrer"])
|
||||
|
||||
event_attrs = %{
|
||||
timestamp: NaiveDateTime.utc_now(),
|
||||
name: params["name"],
|
||||
hostname: strip_www(uri && uri.host),
|
||||
domain: strip_www(params["domain"]) || strip_www(uri && uri.host),
|
||||
pathname: uri && (uri.path || "/"),
|
||||
user_id: generate_user_id(conn, params),
|
||||
country_code: country_code,
|
||||
fingerprint: calculate_fingerprint(conn, params),
|
||||
operating_system: ua && os_name(ua),
|
||||
browser: ua && browser_name(ua),
|
||||
referrer_source: params["source"] || referrer_source(ref),
|
||||
@ -62,18 +53,16 @@ defmodule PlausibleWeb.Api.ExternalController do
|
||||
screen_size: calculate_screen_size(params["screen_width"])
|
||||
}
|
||||
|
||||
changeset = Plausible.Event.changeset(%Plausible.Event{}, event_attrs)
|
||||
changeset = Plausible.ClickhouseEvent.changeset(%Plausible.ClickhouseEvent{}, event_attrs)
|
||||
if changeset.valid? do
|
||||
event = struct(Plausible.ClickhouseEvent, event_attrs)
|
||||
|> Map.put(:timestamp, NaiveDateTime.utc_now())
|
||||
|> Map.put(:user_id, generate_user_id(conn, params))
|
||||
|
||||
session_id = Plausible.Session.Store.on_event(event)
|
||||
|
||||
Map.put(event, :session_id, session_id)
|
||||
|> Plausible.Event.WriteBuffer.insert()
|
||||
else
|
||||
{:error, changeset}
|
||||
end
|
||||
Plausible.Repo.insert(changeset)
|
||||
end
|
||||
end
|
||||
|
||||
@ -86,16 +75,6 @@ defmodule PlausibleWeb.Api.ExternalController do
|
||||
end
|
||||
end
|
||||
|
||||
defp calculate_fingerprint(conn, params) do
|
||||
user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
|
||||
ip_address = List.first(Plug.Conn.get_req_header(conn, "x-bb-ip")) || "" # Netlify sets this header as the remote client IP
|
||||
domain = strip_www(params["domain"]) || ""
|
||||
|
||||
:crypto.hash(:sha256, [user_agent, ip_address, domain])
|
||||
|> Base.encode16
|
||||
|> String.downcase
|
||||
end
|
||||
|
||||
defp generate_user_id(conn, params) do
|
||||
hash_key = Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base) |> binary_part(0, 16)
|
||||
user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
|
||||
|
||||
@ -54,7 +54,6 @@ defmodule PlausibleWeb.Router do
|
||||
pipe_through :api
|
||||
|
||||
post "/event", Api.ExternalController, :event
|
||||
post "/unload", Api.ExternalController, :unload
|
||||
get "/error", Api.ExternalController, :error
|
||||
|
||||
post "/paddle/webhook", Api.PaddleController, :webhook
|
||||
|
||||
@ -1,23 +1,23 @@
|
||||
defmodule Plausible.Workers.FetchTweets do
|
||||
use Plausible.Repo
|
||||
use Oban.Worker, queue: :fetch_tweets
|
||||
alias Plausible.Clickhouse
|
||||
alias Plausible.Twitter.Tweet
|
||||
@oauth_credentials Application.get_env(:plausible, :twitter, %{}) |> OAuther.credentials()
|
||||
use Oban.Worker, queue: :fetch_tweets
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(_args, _job) do
|
||||
new_links = Repo.all(
|
||||
from e in Plausible.Event,
|
||||
where: e.timestamp > fragment("(now() - '6 days'::interval)") and e.timestamp < fragment("(now() - '5 days'::interval)"),
|
||||
or_where: e.timestamp > fragment("(now() - '1 days'::interval)"),
|
||||
def perform(_args, _job, twitter_api \\ Plausible.Twitter.Api) do
|
||||
new_links = Clickhouse.all(
|
||||
from e in Plausible.ClickhouseEvent,
|
||||
where: e.timestamp > fragment("(now() - INTERVAL 6 day)") and e.timestamp < fragment("(now() - INTERVAL 5 day)"),
|
||||
or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"),
|
||||
where: e.referrer_source == "Twitter",
|
||||
where: e.referrer not in ["t.co", "t.co/"],
|
||||
distinct: true,
|
||||
select: e.referrer
|
||||
)
|
||||
) |> Enum.map(fn event -> event["referrer"] end)
|
||||
|
||||
for link <- new_links do
|
||||
results = search(link)
|
||||
results = twitter_api.search(link)
|
||||
|
||||
for tweet <- results do
|
||||
{:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
|
||||
@ -48,13 +48,4 @@ defmodule Plausible.Workers.FetchTweets do
|
||||
String.replace(text, "@" <> mention["screen_name"], html)
|
||||
end)
|
||||
end
|
||||
|
||||
defp search(link) do
|
||||
params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
|
||||
params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, @oauth_credentials)
|
||||
uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
|
||||
response = HTTPoison.get!(uri)
|
||||
Jason.decode!(response.body)
|
||||
|> Map.get("statuses")
|
||||
end
|
||||
end
|
||||
|
||||
@ -4,17 +4,7 @@ defmodule Plausible.Repo.Migrations.AddNameToEvents do
|
||||
|
||||
def change do
|
||||
alter table(:events) do
|
||||
add :name, :string
|
||||
end
|
||||
|
||||
flush()
|
||||
|
||||
Repo.update_all(Plausible.Event, set: [name: "pageview"])
|
||||
|
||||
flush()
|
||||
|
||||
alter table(:events) do
|
||||
modify :name, :string, null: false
|
||||
add :name, :string, null: false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -1,80 +0,0 @@
|
||||
defmodule Plausible.Ingest.FingerprintSessionTest do
|
||||
use Plausible.DataCase
|
||||
alias Plausible.Ingest
|
||||
|
||||
defp capture_session(fingerprint) do
|
||||
session_pid = :global.whereis_name(fingerprint)
|
||||
Process.monitor(session_pid)
|
||||
|
||||
assert_receive({:DOWN, session_pid, :process, _, :normal})
|
||||
|
||||
Repo.one(Plausible.FingerprintSession)
|
||||
end
|
||||
|
||||
describe "on_event/1" do
|
||||
test "starts a new session if there is no session for user id" do
|
||||
pageview = insert(:pg_pageview)
|
||||
|
||||
refute is_pid(:global.whereis_name(pageview.fingerprint))
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
|
||||
assert is_pid(:global.whereis_name(pageview.fingerprint))
|
||||
end
|
||||
|
||||
test "copies event data to session" do
|
||||
pageview = insert(:pg_pageview)
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
|
||||
session = capture_session(pageview.fingerprint)
|
||||
|
||||
assert session.fingerprint == pageview.fingerprint
|
||||
assert session.start == pageview.timestamp
|
||||
end
|
||||
|
||||
test "inserts bounced session when timeout fires after one pageview" do
|
||||
pageview = insert(:pg_pageview)
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
|
||||
session = capture_session(pageview.fingerprint)
|
||||
assert session.is_bounce
|
||||
end
|
||||
|
||||
test "session with two events is not a bounce" do
|
||||
pageview = insert(:pg_pageview)
|
||||
pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint)
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
Ingest.FingerprintSession.on_event(pageview2)
|
||||
|
||||
session = capture_session(pageview.fingerprint)
|
||||
refute session.is_bounce
|
||||
end
|
||||
|
||||
test "captures the exit page" do
|
||||
pageview = insert(:pg_pageview)
|
||||
pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint, pathname: "/exit")
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
Ingest.FingerprintSession.on_event(pageview2)
|
||||
|
||||
session = capture_session(pageview.fingerprint)
|
||||
assert session.exit_page == "/exit"
|
||||
end
|
||||
end
|
||||
|
||||
describe "on_unload/1" do
|
||||
test "uses the unload timestamp to calculate session length" do
|
||||
pageview = insert(:pg_pageview)
|
||||
unload_timestamp = Timex.shift(pageview.timestamp, seconds: 30)
|
||||
|
||||
Ingest.FingerprintSession.on_event(pageview)
|
||||
Ingest.FingerprintSession.on_unload(pageview.fingerprint, unload_timestamp)
|
||||
|
||||
session = capture_session(pageview.fingerprint)
|
||||
assert session.length == 30
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -2,11 +2,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
use PlausibleWeb.ConnCase
|
||||
use Plausible.Repo
|
||||
|
||||
defp finalize_session(fingerprint) do
|
||||
session_pid = :global.whereis_name(fingerprint)
|
||||
Process.monitor(session_pid)
|
||||
defp get_event(domain) do
|
||||
Plausible.Event.WriteBuffer.flush()
|
||||
|
||||
assert_receive({:DOWN, session_pid, _, _, _})
|
||||
events = Plausible.Clickhouse.all(
|
||||
from e in Plausible.ClickhouseEvent,
|
||||
where: e.domain ==^domain,
|
||||
order_by: [desc: e.timestamp],
|
||||
limit: 1
|
||||
)
|
||||
List.first(events)
|
||||
end
|
||||
|
||||
@user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
|
||||
@ -15,6 +20,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
describe "POST /api/event" do
|
||||
test "records the event", %{conn: conn} do
|
||||
params = %{
|
||||
domain: "external-controller-test-1.com",
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "http://m.facebook.com/",
|
||||
@ -27,93 +33,68 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("x-country", @country_code)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-1.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.hostname == "gigride.live"
|
||||
assert pageview.domain == "gigride.live"
|
||||
assert pageview.pathname == "/"
|
||||
assert pageview.country_code == @country_code
|
||||
end
|
||||
|
||||
test "can specify the domain", %{conn: conn} do
|
||||
params = %{
|
||||
name: "custom event",
|
||||
url: "http://gigride.live/",
|
||||
domain: "some_site.com",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
}
|
||||
|
||||
conn = conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
event = Repo.one(Plausible.Event)
|
||||
finalize_session(event.fingerprint)
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert event.domain == "some_site.com"
|
||||
assert pageview["hostname"] == "gigride.live"
|
||||
assert pageview["domain"] == "external-controller-test-1.com"
|
||||
assert pageview["pathname"] == "/"
|
||||
assert pageview["country_code"] == @country_code
|
||||
end
|
||||
|
||||
test "www. is stripped from domain", %{conn: conn} do
|
||||
params = %{
|
||||
name: "custom event",
|
||||
url: "http://gigride.live/",
|
||||
domain: "www.some_site.com",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "www.external-controller-test-2.com"
|
||||
}
|
||||
|
||||
conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-2.com")
|
||||
|
||||
assert pageview.domain == "some_site.com"
|
||||
assert pageview["domain"] == "external-controller-test-2.com"
|
||||
end
|
||||
|
||||
test "www. is stripped from hostname", %{conn: conn} do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://www.example.com/"
|
||||
url: "http://www.example.com/",
|
||||
domain: "external-controller-test-3.com"
|
||||
}
|
||||
|
||||
conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-3.com")
|
||||
|
||||
assert pageview.hostname == "example.com"
|
||||
assert pageview["hostname"] == "example.com"
|
||||
end
|
||||
|
||||
test "empty path defaults to /", %{conn: conn} do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://www.example.com"
|
||||
url: "http://www.example.com",
|
||||
domain: "external-controller-test-4.com"
|
||||
}
|
||||
|
||||
conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-4.com")
|
||||
|
||||
assert pageview.pathname == "/"
|
||||
assert pageview["pathname"] == "/"
|
||||
end
|
||||
|
||||
test "bots and crawlers are ignored", %{conn: conn} do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://www.example.com/",
|
||||
new_visitor: true
|
||||
domain: "external-controller-test-5.com"
|
||||
}
|
||||
|
||||
conn
|
||||
@ -121,17 +102,14 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", "generic crawler")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageviews = Repo.all(Plausible.Event)
|
||||
|
||||
assert Enum.count(pageviews) == 0
|
||||
assert get_event("external-controller-test-5.com") == nil
|
||||
end
|
||||
|
||||
test "parses user_agent", %{conn: conn} do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-6.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -139,12 +117,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-6.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.operating_system == "Mac"
|
||||
assert pageview.browser == "Chrome"
|
||||
assert pageview["operating_system"] == "Mac"
|
||||
assert pageview["browser"] == "Chrome"
|
||||
end
|
||||
|
||||
test "parses referrer", %{conn: conn} do
|
||||
@ -152,9 +129,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "https://facebook.com",
|
||||
initial_referrer: "https://facebook.com",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-7.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -162,12 +137,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-7.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer_source == "Facebook"
|
||||
assert pageview.initial_referrer_source == "Facebook"
|
||||
assert pageview["referrer_source"] == "Facebook"
|
||||
end
|
||||
|
||||
test "strips trailing slash from referrer", %{conn: conn} do
|
||||
@ -175,9 +148,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "https://facebook.com/page/",
|
||||
initial_referrer: "https://facebook.com/page/",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-8.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -185,14 +156,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-8.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer == "facebook.com/page"
|
||||
assert pageview.initial_referrer == "facebook.com/page"
|
||||
assert pageview.referrer_source == "Facebook"
|
||||
assert pageview.initial_referrer_source == "Facebook"
|
||||
assert pageview["referrer"] == "facebook.com/page"
|
||||
assert pageview["referrer_source"] == "Facebook"
|
||||
end
|
||||
|
||||
test "ignores when referrer is internal", %{conn: conn} do
|
||||
@ -200,9 +168,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "https://gigride.live",
|
||||
initial_referrer: "https://gigride.live",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-9.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -210,12 +176,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-9.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer_source == nil
|
||||
assert pageview.initial_referrer_source == nil
|
||||
assert pageview["referrer_source"] == ""
|
||||
end
|
||||
|
||||
test "ignores localhost referrer", %{conn: conn} do
|
||||
@ -223,9 +187,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "http://localhost:4000/",
|
||||
initial_referrer: "http://localhost:4000/",
|
||||
new_visitor: true,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-10.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -233,12 +195,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-10.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer_source == nil
|
||||
assert pageview.initial_referrer_source == nil
|
||||
assert pageview["referrer_source"] == ""
|
||||
end
|
||||
|
||||
test "parses subdomain referrer", %{conn: conn} do
|
||||
@ -246,9 +206,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "https://blog.gigride.live",
|
||||
initial_referrer: "https://blog.gigride.live",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-11.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -256,12 +214,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-11.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer_source == "blog.gigride.live"
|
||||
assert pageview.initial_referrer_source == "blog.gigride.live"
|
||||
assert pageview["referrer_source"] == "blog.gigride.live"
|
||||
end
|
||||
|
||||
test "referrer is cleaned", %{conn: conn} do
|
||||
@ -269,20 +225,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://www.example.com/",
|
||||
referrer: "https://www.indiehackers.com/page?query=param#hash",
|
||||
initial_referrer: "https://www.indiehackers.com/?query=param#hash",
|
||||
uid: UUID.uuid4(),
|
||||
new_visitor: true
|
||||
domain: "external-controller-test-12.com"
|
||||
}
|
||||
|
||||
conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-12.com")
|
||||
|
||||
assert pageview.referrer == "indiehackers.com/page"
|
||||
assert pageview.initial_referrer == "indiehackers.com"
|
||||
assert pageview["referrer"] == "indiehackers.com/page"
|
||||
end
|
||||
|
||||
test "source param controls the referrer source", %{conn: conn} do
|
||||
@ -291,20 +243,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
url: "http://www.example.com/",
|
||||
referrer: "https://betalist.com/my-produxct",
|
||||
source: "betalist",
|
||||
initial_source: "betalist",
|
||||
uid: UUID.uuid4(),
|
||||
new_visitor: true
|
||||
domain: "external-controller-test-13.com"
|
||||
}
|
||||
|
||||
conn
|
||||
|> put_req_header("content-type", "text/plain")
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-13.com")
|
||||
|
||||
assert pageview.referrer_source == "betalist"
|
||||
assert pageview.initial_referrer_source == "betalist"
|
||||
assert pageview["referrer_source"] == "betalist"
|
||||
end
|
||||
|
||||
test "if it's an :unknown referrer, just the domain is used", %{conn: conn} do
|
||||
@ -312,8 +260,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "https://www.indiehackers.com/landing-page-feedback",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-14.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -321,11 +268,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-14.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.referrer_source == "indiehackers.com"
|
||||
assert pageview["referrer_source"] == "indiehackers.com"
|
||||
end
|
||||
|
||||
test "if the referrer is not http or https, it is ignored", %{conn: conn} do
|
||||
@ -333,8 +279,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
referrer: "android-app://com.google.android.gm",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-15.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -342,11 +287,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-15.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert is_nil(pageview.referrer_source)
|
||||
assert pageview["referrer_source"] == ""
|
||||
end
|
||||
|
||||
end
|
||||
@ -355,9 +299,8 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
new_visitor: true,
|
||||
screen_width: 480,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-16.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -365,19 +308,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-16.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.screen_size == "Mobile"
|
||||
assert pageview["screen_size"] == "Mobile"
|
||||
end
|
||||
|
||||
test "screen size is nil if screen_width is missing", %{conn: conn} do
|
||||
params = %{
|
||||
name: "pageview",
|
||||
url: "http://gigride.live/",
|
||||
new_visitor: true,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-17.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -385,19 +326,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
pageview = Repo.one(Plausible.Event)
|
||||
finalize_session(pageview.fingerprint)
|
||||
pageview = get_event("external-controller-test-17.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert pageview.screen_size == nil
|
||||
assert pageview["screen_size"] == ""
|
||||
end
|
||||
|
||||
test "can trigger a custom event", %{conn: conn} do
|
||||
params = %{
|
||||
name: "custom event",
|
||||
url: "http://gigride.live/",
|
||||
new_visitor: false,
|
||||
uid: UUID.uuid4()
|
||||
domain: "external-controller-test-18.com"
|
||||
}
|
||||
|
||||
conn = conn
|
||||
@ -405,11 +344,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|
||||
|> put_req_header("user-agent", @user_agent)
|
||||
|> post("/api/event", Jason.encode!(params))
|
||||
|
||||
event = Repo.one(Plausible.Event)
|
||||
finalize_session(event.fingerprint)
|
||||
event = get_event("external-controller-test-18.com")
|
||||
|
||||
assert response(conn, 202) == ""
|
||||
assert event.name == "custom event"
|
||||
assert event["name"] == "custom event"
|
||||
end
|
||||
|
||||
test "responds 400 when required fields are missing", %{conn: conn} do
|
||||
|
||||
@ -98,7 +98,9 @@ defmodule Plausible.Test.ClickhouseSetup do
|
||||
%{name: "pageview", domain: "test-site.com", timestamp: Timex.now() |> Timex.shift(minutes: -6)},
|
||||
|
||||
%{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},
|
||||
%{name: "pageview", domain: "public-site.io"}
|
||||
%{name: "pageview", domain: "public-site.io"},
|
||||
%{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/a-link", referrer_source: "Twitter"},
|
||||
%{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/b-link", referrer_source: "Twitter", timestamp: Timex.now() |> Timex.shift(days: -5)}
|
||||
])
|
||||
|
||||
Plausible.TestUtils.create_sessions([
|
||||
|
||||
@ -43,40 +43,6 @@ defmodule Plausible.Factory do
|
||||
}
|
||||
end
|
||||
|
||||
def session_factory do
|
||||
hostname = sequence(:domain, &"example-#{&1}.com")
|
||||
|
||||
%Plausible.FingerprintSession{
|
||||
hostname: hostname,
|
||||
domain: hostname,
|
||||
entry_page: "/",
|
||||
fingerprint: UUID.uuid4(),
|
||||
start: Timex.now(),
|
||||
is_bounce: false
|
||||
}
|
||||
end
|
||||
|
||||
def pg_pageview_factory do
|
||||
struct!(
|
||||
pg_event_factory(),
|
||||
%{
|
||||
name: "pageview"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def pg_event_factory do
|
||||
hostname = sequence(:domain, &"example-#{&1}.com")
|
||||
|
||||
%Plausible.Event{
|
||||
hostname: hostname,
|
||||
domain: hostname,
|
||||
pathname: "/",
|
||||
timestamp: Timex.now(),
|
||||
fingerprint: UUID.uuid4()
|
||||
}
|
||||
end
|
||||
|
||||
def pageview_factory do
|
||||
struct!(
|
||||
event_factory(),
|
||||
|
||||
@ -1,6 +1,52 @@
|
||||
defmodule Plausible.Workers.FetchTweetsTest do
|
||||
use Plausible.DataCase
|
||||
alias Plausible.Workers.FetchTweets
|
||||
import Double
|
||||
|
||||
test "fetches Twitter referrals from the last day" do
|
||||
twitter_mock = stub(Plausible.Twitter.Api, :search, fn(_link) -> [] end)
|
||||
FetchTweets.perform(nil, nil, twitter_mock)
|
||||
|
||||
assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]})
|
||||
end
|
||||
|
||||
test "fetches Twitter referrals from 5-6 days ago" do
|
||||
twitter_mock = stub(Plausible.Twitter.Api, :search, fn(_link) -> [] end)
|
||||
FetchTweets.perform(nil, nil, twitter_mock)
|
||||
|
||||
assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]})
|
||||
end
|
||||
|
||||
test "stores twitter results" do
|
||||
tweet = %{
|
||||
"full_text" => "a Tweet body",
|
||||
"id_str" => "the_tweet_id",
|
||||
"created_at" => "Mon May 06 20:01:29 +0000 2019",
|
||||
"user" => %{
|
||||
"screen_name" => "twitter_author",
|
||||
"name" => "Twitter Author",
|
||||
"profile_image_url_https" => "https://image.com"
|
||||
},
|
||||
"entities" => %{
|
||||
"user_mentions" => [],
|
||||
"urls" => []
|
||||
}
|
||||
}
|
||||
|
||||
twitter_mock = stub(Plausible.Twitter.Api, :search, fn
|
||||
("t.co/a-link") -> [tweet]
|
||||
(_link) -> []
|
||||
end)
|
||||
FetchTweets.perform(nil, nil, twitter_mock)
|
||||
|
||||
[found_tweet] = Repo.all(from t in Plausible.Twitter.Tweet)
|
||||
assert found_tweet.tweet_id == "the_tweet_id"
|
||||
assert found_tweet.text == "a Tweet body"
|
||||
assert found_tweet.author_handle == "twitter_author"
|
||||
assert found_tweet.author_name == "Twitter Author"
|
||||
assert found_tweet.author_image == "https://image.com"
|
||||
assert found_tweet.created == ~N[2019-05-06 20:01:29]
|
||||
end
|
||||
|
||||
describe "processing tweet entities" do
|
||||
test "inlines links to the body" do
|
||||
|
||||
Loading…
Reference in New Issue
Block a user